From bec9235b0a4bb9dc4c8fcb0cf05338115dab593f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 21 Feb 2025 10:00:47 -0500 Subject: [PATCH 01/74] build: fix makefile rules and windows path separator used in build (#57472) Closes #45262 (cherry picked from commit e026affc3ad49ac199cf22ce05aeb412d50fcb21) --- Make.inc | 81 +++++++++++++++++++++++------------------ Makefile | 16 ++++---- base/Makefile | 87 +++++++++++++++++++------------------------- cli/Makefile | 4 +- src/Makefile | 8 ++-- src/flisp/Makefile | 8 ++-- src/support/Makefile | 8 ++-- 7 files changed, 106 insertions(+), 106 deletions(-) diff --git a/Make.inc b/Make.inc index 16e238c6f0683..466e97cc43c4f 100644 --- a/Make.inc +++ b/Make.inc @@ -111,6 +111,11 @@ endef COMMA:=, SPACE:=$(eval) $(eval) +# define various helper macros for safe interpolation into various parsers +shell_escape='$(subst ','\'',$1)' +c_escape="$(subst ",\",$(subst \,\\,$1))" +julia_escape=$(call c_escape,$1) + # force a sane / stable configuration export LC_ALL=C export LANG=C @@ -644,12 +649,12 @@ CXX += -Qunused-arguments export CCACHE_CPP2 := yes endif else #USECCACHE -CC_BASE := $(shell echo $(CC) | cut -d' ' -f1) -CC_ARG := $(shell echo $(CC) | cut -s -d' ' -f2-) -CXX_BASE := $(shell echo $(CXX) | cut -d' ' -f1) -CXX_ARG := $(shell echo $(CXX) | cut -s -d' ' -f2-) -FC_BASE := $(shell echo $(FC) 2>/dev/null | cut -d' ' -f1) -FC_ARG := $(shell echo $(FC) 2>/dev/null | cut -s -d' ' -f2-) +CC_BASE := $(shell printf "%s\n" $(call shell_escape,$(CC)) | cut -d' ' -f1) +CC_ARG := $(shell printf "%s\n" $(call shell_escape,$(CC)) | cut -s -d' ' -f2-) +CXX_BASE := $(shell printf "%s\n" $(call shell_escape,$(CXX)) | cut -d' ' -f1) +CXX_ARG := $(shell printf "%s\n" $(call shell_escape,$(CXX)) | cut -s -d' ' -f2-) +FC_BASE := $(shell printf "%s\n" $(call shell_escape,$(FC)) 2>/dev/null | cut -d' ' -f1) +FC_ARG := $(shell printf "%s\n" $(call shell_escape,$(FC)) 2>/dev/null | cut -s -d' ' -f2-) endif JFFLAGS := -O2 $(fPIC) @@ -776,7 +781,7 @@ LDFLAGS += -L$(build_libdir) -Wl,-rpath,$(build_libdir) endif # gfortran endif # FreeBSD -ifneq ($(CC_BASE)$(CXX_BASE),$(shell echo $(CC) | cut -d' ' -f1)$(shell echo $(CXX) | cut -d' ' -f1)) +ifneq ($(CC_BASE)$(CXX_BASE),$(shell printf "%s\n" $(call shell_escape,$(CC)) | cut -d' ' -f1)$(shell printf "%s\n" $(call shell_escape,$(CXX)) | cut -d' ' -f1)) $(error Forgot override directive on CC or CXX in Make.user? Cowardly refusing to build) endif @@ -1663,6 +1668,12 @@ $(subst /,\\,$(subst $(shell $(2) pwd),$(shell $(2) cmd //C cd),$(abspath $(1))) endef endif +ifeq ($(OS), WINNT) +normalize_path = $(subst /,\,$1) +else +normalize_path = $1 +endif + define symlink_target # (from, to-dir, to-name) CLEAN_TARGETS += clean-$$(abspath $(2)/$(3)) clean-$$(abspath $(2)/$(3)): @@ -1729,20 +1740,20 @@ JULIA_SYSIMG_release := $(build_private_libdir)/sys.$(SHLIB_EXT) JULIA_SYSIMG := $(JULIA_SYSIMG_$(JULIA_BUILD_MODE)) define dep_lib_path -$(shell $(PYTHON) $(call python_cygpath,$(JULIAHOME)/contrib/relative_path.py) $(1) $(2)) +$(call normalize_path,$(shell $(PYTHON) $(call python_cygpath,$(JULIAHOME)/contrib/relative_path.py) $(1) $(2))) endef -LIBJULIAINTERNAL_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIAINTERNAL_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-internal.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIAINTERNAL_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/libjulia-internal.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIAINTERNAL_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libjulia-internal.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIAINTERNAL_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIAINTERNAL_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIAINTERNAL_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIAINTERNAL_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libjulia-internal-debug.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIACODEGEN_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-codegen.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIACODEGEN_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-codegen.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIACODEGEN_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/libjulia-codegen.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIACODEGEN_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libjulia-codegen.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIACODEGEN_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_SHLIB_EXT)) -LIBJULIACODEGEN_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIACODEGEN_DEBUG_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_SHLIB_EXT)) +LIBJULIACODEGEN_DEBUG_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/libjulia-codegen-debug.$(JL_MAJOR_SHLIB_EXT)) ifeq ($(OS),WINNT) ifeq ($(BINARY),32) @@ -1770,34 +1781,34 @@ endif # USE_SYSTEM_CSL causes it to get symlinked into build_private_shlibdir ifeq ($(USE_SYSTEM_CSL),1) -LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_private_shlibdir)/$(LIBGCC_NAME)) +LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_private_shlibdir)/$(LIBGCC_NAME)) else -LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/$(LIBGCC_NAME)) +LIBGCC_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/$(LIBGCC_NAME)) endif -LIBGCC_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBGCC_NAME)) +LIBGCC_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/$(LIBGCC_NAME)) # We only bother to define this on Linux, as that's the only platform that does libstdc++ probing # On all other platforms, the LIBSTDCXX_*_DEPLIB variables will be empty. ifeq ($(OS),Linux) LIBSTDCXX_NAME := libstdc++.so.6 ifeq ($(USE_SYSTEM_CSL),1) -LIBSTDCXX_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_private_shlibdir)/$(LIBSTDCXX_NAME)) +LIBSTDCXX_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_private_shlibdir)/$(LIBSTDCXX_NAME)) else -LIBSTDCXX_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/$(LIBSTDCXX_NAME)) +LIBSTDCXX_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/$(LIBSTDCXX_NAME)) endif -LIBSTDCXX_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBSTDCXX_NAME)) +LIBSTDCXX_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/$(LIBSTDCXX_NAME)) endif # USE_SYSTEM_LIBM and USE_SYSTEM_OPENLIBM causes it to get symlinked into build_private_shlibdir ifeq ($(USE_SYSTEM_LIBM),1) -LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) +LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) else ifeq ($(USE_SYSTEM_OPENLIBM),1) -LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) +LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) else -LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_libdir),$(build_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) +LIBM_BUILD_DEPLIB := $(call dep_lib_path,$(build_shlibdir),$(build_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) endif -LIBM_INSTALL_DEPLIB := $(call dep_lib_path,$(libdir),$(private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) +LIBM_INSTALL_DEPLIB := $(call dep_lib_path,$(shlibdir),$(private_shlibdir)/$(LIBMNAME).$(SHLIB_EXT)) # We list: # * libgcc_s, because FreeBSD needs to load ours, not the system one. @@ -1861,7 +1872,7 @@ ifeq ($(VERBOSE), 0) QUIET_MAKE = -s -GOAL=$(subst ','\'',$(subst $(abspath $(JULIAHOME))/,,$(abspath $@))) +GOAL=$(call shell_escape,$(subst $(abspath $(JULIAHOME))/,,$(abspath $@))) PRINT_CC = printf ' %b %b\n' $(CCCOLOR)CC$(ENDCOLOR) $(SRCCOLOR)$(GOAL)$(ENDCOLOR); $(1) PRINT_ANALYZE = printf ' %b %b\n' $(CCCOLOR)ANALYZE$(ENDCOLOR) $(SRCCOLOR)$(GOAL)$(ENDCOLOR); $(1) @@ -1873,13 +1884,13 @@ PRINT_DTRACE = printf ' %b %b\n' $(DTRACECOLOR)DTRACE$(ENDCOLOR) $(BINCOLOR)$ else QUIET_MAKE = -PRINT_CC = echo '$(subst ','\'',$(1))'; $(1) -PRINT_ANALYZE = echo '$(subst ','\'',$(1))'; $(1) -PRINT_LINK = echo '$(subst ','\'',$(1))'; $(1) -PRINT_PERL = echo '$(subst ','\'',$(1))'; $(1) -PRINT_FLISP = echo '$(subst ','\'',$(1))'; $(1) -PRINT_JULIA = echo '$(subst ','\'',$(1))'; $(1) -PRINT_DTRACE = echo '$(subst ','\'',$(1))'; $(1) +PRINT_CC = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_ANALYZE = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_LINK = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_PERL = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_FLISP = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_JULIA = printf "%s\n" $(call shell_escape,$(1)); $(1) +PRINT_DTRACE = printf "%s\n" $(call shell_escape,$(1)); $(1) endif # VERBOSE @@ -1887,4 +1898,4 @@ endif # VERBOSE # call print-VARIABLE to see the runtime value of any variable # (hardened against any special characters appearing in the output) print-%: - @echo '$*=$(subst ','\'',$(subst $(newline),\n,$($*)))' + @printf "%s\n" $(call shell_escape,$*)=$(call shell_escape,$(subst $(newline),\n,$($*))) diff --git a/Makefile b/Makefile index 0f1e8c45edf40..a7562946c1699 100644 --- a/Makefile +++ b/Makefile @@ -4,18 +4,18 @@ include $(JULIAHOME)/Make.inc include $(JULIAHOME)/deps/llvm-ver.make # Make sure the user didn't try to build in a path that will confuse the shell or make -METACHARACTERS := [][?*{}() $$%:;&|!\#,\\`\":]\|/\./\|/\.\./ +METACHARACTERS := [][?*{}() $$%:;&|!\#,\\`\": ]\|/\./\|/\.\./ ifneq (,$(findstring ',$(value BUILDROOT))) $(error cowardly refusing to build into directory with a single-quote in the path) endif ifneq (,$(findstring ',$(value JULIAHOME))) $(error cowardly refusing to build from source directory with a single-quote in the path) endif -ifneq (,$(shell echo '$(value BUILDROOT)/' | grep '$(METACHARACTERS)')) +ifneq (,$(shell printf "%s\n" $(call shell_escape,$(value BUILDROOT)/) | grep '$(METACHARACTERS)')) $(error cowardly refusing to build into directory with a shell-metacharacter in the path\ (got: $(value BUILDROOT))) endif -ifneq (,$(shell echo '$(value JULIAHOME)/' | grep '$(METACHARACTERS)')) +ifneq (,$(shell printf "%s\n" $(call shell_escape,$(value JULIAHOME)/) | grep '$(METACHARACTERS)')) $(error cowardly refusing to build from source directory with a shell-metacharacter in the path\ (got: $(value JULIAHOME))) endif @@ -33,9 +33,9 @@ BUILDDIRMAKE := $(addsuffix /Makefile,$(BUILDDIRS)) $(BUILDROOT)/sysimage.mk $(B DIRS += $(BUILDDIRS) $(BUILDDIRMAKE): | $(BUILDDIRS) @# add Makefiles to the build directories for convenience (pointing back to the source location of each) - @echo '# -- This file is automatically generated in julia/Makefile -- #' > $@ - @echo 'BUILDROOT=$(BUILDROOT)' >> $@ - @echo 'include $(JULIAHOME)$(patsubst $(BUILDROOT)%,%,$@)' >> $@ + @printf "%s\n" '# -- This file is automatically generated in julia/Makefile -- #' > $@ + @printf "%s\n" 'BUILDROOT=$(BUILDROOT)' >> $@ + @printf "%s\n" 'include $(JULIAHOME)$(patsubst $(BUILDROOT)%,%,$@)' >> $@ julia-deps: | $(BUILDDIRMAKE) configure-y: | $(BUILDDIRMAKE) configure: @@ -67,7 +67,7 @@ $(BUILDROOT)/doc/_build/html/en/index.html: $(shell find $(BUILDROOT)/base $(BUI julia-symlink: julia-cli-$(JULIA_BUILD_MODE) ifeq ($(OS),WINNT) - echo '@"%~dp0/'"$$(echo '$(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE))')"'" %*' | tr / '\\' > $(BUILDROOT)/julia.bat + printf '@"%%~dp0/%s" %%*\n' "$$(printf "%s\n" '$(call rel_path,$(BUILDROOT),$(JULIA_EXECUTABLE))')" | tr / '\\' > $(BUILDROOT)/julia.bat chmod a+x $(BUILDROOT)/julia.bat else ifndef JULIA_VAGRANT_BUILD @@ -306,7 +306,7 @@ endif # Note that we disable MSYS2's path munging here, as otherwise # it replaces our `:`-separated list as a `;`-separated one. define stringreplace - MSYS2_ARG_CONV_EXCL='*' $(build_depsbindir)/stringreplace $$(strings -t x - '$1' | grep "$2" | awk '{print $$1;}') "$3" 255 "$(call cygpath_w,$1)" + MSYS2_ARG_CONV_EXCL='*' $(build_depsbindir)/stringreplace $$(strings -t x - '$1' | grep '$2' | awk '{print $$1;}') '$3' 255 '$(call cygpath_w,$1)' endef diff --git a/base/Makefile b/base/Makefile index 09f79e5b98611..34791f7b4b0d4 100644 --- a/base/Makefile +++ b/base/Makefile @@ -18,9 +18,9 @@ else endif define parse_features -@echo "# $(2) features" >> $@ +@printf "%s\n" "# $(2) features" >> $@ @$(call PRINT_PERL, cat $(SRCDIR)/../src/features_$(1).h | perl -lne 'print "const JL_$(2)_$$1 = UInt32($$2)" if /^\s*JL_FEATURE_DEF(?:_NAME)?\(\s*(\w+)\s*,\s*([^,]+)\s*,.*\)\s*(?:\/\/.*)?$$/' >> $@) -@echo >> $@ +@printf "\n" >> $@ endef $(BUILDDIR)/features_h.jl: $(SRCDIR)/../src/features_x86.h $(SRCDIR)/../src/features_aarch32.h $(SRCDIR)/../src/features_aarch64.h @@ -33,7 +33,7 @@ $(BUILDDIR)/pcre_h.jl: $(PCRE_INCL_PATH) @$(call PRINT_PERL, $(CPP) -D PCRE2_CODE_UNIT_WIDTH=8 -dM $< | perl -nle '/^\s*#define\s+PCRE2_(\w*)\s*\(?($(PCRE_CONST))\)?u?\s*$$/ and print index($$1, "ERROR_") == 0 ? "const $$1 = Cint($$2)" : "const $$1 = UInt32($$2)"' | LC_ALL=C sort > $@) $(BUILDDIR)/errno_h.jl: - @$(call PRINT_PERL, echo '#include ' | $(CPP) -dM - | perl -nle 'print "const $$1 = Int32($$2)" if /^#define\s+(E\w+)\s+(\d+)\s*$$/' | LC_ALL=C sort > $@) + @$(call PRINT_PERL, printf "%s\n" '#include ' | $(CPP) -dM - | perl -nle 'print "const $$1 = Int32($$2)" if /^#define\s+(E\w+)\s+(\d+)\s*$$/' | LC_ALL=C sort > $@) $(BUILDDIR)/file_constants.jl: $(SRCDIR)/../src/file_constants.h @$(call PRINT_PERL, $(CPP_STDOUT) -DJULIA $< | perl -nle 'print "$$1 0o$$2" if /^(\s*const\s+[A-z_]+\s+=)\s+(0[0-9]*)\s*$$/; print "$$1" if /^\s*(const\s+[A-z_]+\s+=\s+([1-9]|0x)[0-9A-z]*)\s*$$/' > $@) @@ -42,57 +42,46 @@ $(BUILDDIR)/uv_constants.jl: $(SRCDIR)/../src/uv_constants.h $(LIBUV_INC)/uv/err @$(call PRINT_PERL, $(CPP_STDOUT) "-I$(LIBUV_INC)" -DJULIA $< | tail -n 16 > $@) $(BUILDDIR)/build_h.jl.phony: - @echo "# This file is automatically generated in base/Makefile" > $@ + @printf "%s\n" "# This file is automatically generated in base/Makefile" > $@ ifeq ($(XC_HOST),) - @echo "const MACHINE = \"$(BUILD_MACHINE)\"" >> $@ + @printf "%s\n" "const MACHINE = \"$(BUILD_MACHINE)\"" >> $@ else - @echo "const MACHINE = \"$(XC_HOST)\"" >> $@ + @printf "%s\n" "const MACHINE = \"$(XC_HOST)\"" >> $@ endif - @echo "const libm_name = \"$(LIBMNAME)\"" >> $@ + @printf "%s\n" "const libm_name = \"$(LIBMNAME)\"" >> $@ ifeq ($(USE_BLAS64), 1) - @echo "const USE_BLAS64 = true" >> $@ + @printf "%s\n" "const USE_BLAS64 = true" >> $@ else - @echo "const USE_BLAS64 = false" >> $@ + @printf "%s\n" "const USE_BLAS64 = false" >> $@ endif ifeq ($(USE_GPL_LIBS), 1) - @echo "const USE_GPL_LIBS = true" >> $@ + @printf "%s\n" "const USE_GPL_LIBS = true" >> $@ else - @echo "const USE_GPL_LIBS = false" >> $@ -endif - @echo "const libllvm_version_string = \"$$($(LLVM_CONFIG_HOST) --version)\"" >> $@ - @echo "const libllvm_name = \"$(LLVM_SHARED_LIB_NAME)\"" >> $@ - @echo "const VERSION_STRING = \"$(JULIA_VERSION)\"" >> $@ - @echo "const TAGGED_RELEASE_BANNER = \"$(TAGGED_RELEASE_BANNER)\"" >> $@ -ifeq ($(OS),WINNT) - @printf 'const SYSCONFDIR = "%s"\n' '$(subst /,\\,$(sysconfdir_rel))' >> $@ - @printf 'const DATAROOTDIR = "%s"\n' '$(subst /,\\,$(datarootdir_rel))' >> $@ - @printf 'const DOCDIR = "%s"\n' '$(subst /,\\,$(docdir_rel))' >> $@ - @printf 'const LIBDIR = "%s"\n' '$(subst /,\\,$(libdir_rel))' >> $@ - @printf 'const LIBEXECDIR = "%s"\n' '$(subst /,\\,$(libexecdir_rel))' >> $@ - @printf 'const PRIVATE_LIBDIR = "%s"\n' '$(subst /,\\,$(private_libdir_rel))' >> $@ - @printf 'const PRIVATE_LIBEXECDIR = "%s"\n' '$(subst /,\\,$(private_libexecdir_rel))' >> $@ - @printf 'const INCLUDEDIR = "%s"\n' '$(subst /,\\,$(includedir_rel))' >> $@ -else - @echo "const SYSCONFDIR = \"$(sysconfdir_rel)\"" >> $@ - @echo "const DATAROOTDIR = \"$(datarootdir_rel)\"" >> $@ - @echo "const DOCDIR = \"$(docdir_rel)\"" >> $@ - @echo "const LIBDIR = \"$(libdir_rel)\"" >> $@ - @echo "const LIBEXECDIR = \"$(libexecdir_rel)\"" >> $@ - @echo "const PRIVATE_LIBDIR = \"$(private_libdir_rel)\"" >> $@ - @echo "const PRIVATE_LIBEXECDIR = \"$(private_libexecdir_rel)\"" >> $@ - @echo "const INCLUDEDIR = \"$(includedir_rel)\"" >> $@ + @printf "%s\n" "const USE_GPL_LIBS = false" >> $@ endif + @printf "%s\n" "const libllvm_version_string = \"$$($(LLVM_CONFIG_HOST) --version)\"" >> $@ + @printf "%s\n" "const libllvm_name = \"$(LLVM_SHARED_LIB_NAME)\"" >> $@ + @printf "%s\n" "const VERSION_STRING = \"$(JULIA_VERSION)\"" >> $@ + @printf "%s\n" "const TAGGED_RELEASE_BANNER = \"$(TAGGED_RELEASE_BANNER)\"" >> $@ + @printf "%s\n" "const SYSCONFDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(sysconfdir_rel)))) >> $@ + @printf "%s\n" "const DATAROOTDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(datarootdir_rel)))) >> $@ + @printf "%s\n" "const DOCDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(docdir_rel)))) >> $@ + @printf "%s\n" "const LIBDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(libdir_rel)))) >> $@ + @printf "%s\n" "const LIBEXECDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(libexecdir_rel)))) >> $@ + @printf "%s\n" "const PRIVATE_LIBDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(private_libdir_rel)))) >> $@ + @printf "%s\n" "const PRIVATE_LIBEXECDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(private_libexecdir_rel)))) >> $@ + @printf "%s\n" "const INCLUDEDIR = "$(call shell_escape,$(call julia_escape,$(call normalize_path,$(includedir_rel)))) >> $@ ifeq ($(DARWIN_FRAMEWORK), 1) - @echo "const DARWIN_FRAMEWORK = true" >> $@ - @echo "const DARWIN_FRAMEWORK_NAME = \"$(FRAMEWORK_NAME)\"" >> $@ + @printf "%s\n" "const DARWIN_FRAMEWORK = true" >> $@ + @printf "%s\n" "const DARWIN_FRAMEWORK_NAME = \"$(FRAMEWORK_NAME)\"" >> $@ else - @echo "const DARWIN_FRAMEWORK = false" >> $@ + @printf "%s\n" "const DARWIN_FRAMEWORK = false" >> $@ endif ifeq ($(OS), Darwin) - @echo "const MACOS_PRODUCT_VERSION = \"$(shell sw_vers -productVersion)\"" >> $@ - @echo "const MACOS_PLATFORM_VERSION = \"$(shell xcrun --show-sdk-version)\"" >> $@ + @printf "%s\n" "const MACOS_PRODUCT_VERSION = \"$(shell sw_vers -productVersion)\"" >> $@ + @printf "%s\n" "const MACOS_PLATFORM_VERSION = \"$(shell xcrun --show-sdk-version)\"" >> $@ endif - @echo "const BUILD_TRIPLET = \"$(BB_TRIPLET_LIBGFORTRAN_CXXABI)\"" >> $@ + @printf "%s\n" "const BUILD_TRIPLET = \"$(BB_TRIPLET_LIBGFORTRAN_CXXABI)\"" >> $@ @# This to ensure that we always rebuild this file, but only when it is modified do we touch build_h.jl, @# ensuring we rebuild the system image as infrequently as possible @@ -115,10 +104,10 @@ ifneq ($(NO_GIT), 1) rm -f $@; \ fi else -ifeq ($(shell [ -f $(BUILDDIR)/version_git.jl ] && echo "true"), true) +ifeq ($(shell [ -f $(BUILDDIR)/version_git.jl ] && printf "true\n"), true) @# Give warning if boilerplate git is used @if grep -q "Default output if git is not available" $(BUILDDIR)/version_git.jl; then \ - echo "WARNING: Using boilerplate git version info" >&2; \ + printf "WARNING: Using boilerplate git version info\n" >&2; \ fi else $(warning "WARNING: Generating boilerplate git version info") @@ -141,7 +130,7 @@ resolve_path = \ if [ -n "$${$1_}" ]; then $1_wd=`dirname "$${$1}"`; $1="$${$1_}"; fi ## if it's a relative path, make it an absolute path resolve_path += && \ - if [ -z "`echo $${$1} | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi + if [ -z "`printf "%s\n" "$${$1}" | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi ifeq ($(OS), Darwin) # try to use the install_name id instead (unless it is an @rpath or such) # if it's a relative path, make it an absolute path using the working directory from $1, @@ -150,7 +139,7 @@ resolve_path += && \ $1_=`otool -D $${$1} | tail -n +2 | sed -e 's/^@.*$$//'` && \ if [ -n "$${$1_}" ]; then \ $1_wd=`dirname "$${$1}"`; $1=$${$1_}; \ - if [ -z "`echo $${$1} | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi; \ + if [ -z "`printf "%s\n" $${$1} | grep '^/'`" ]; then $1=$${$1_wd}/$${$1}; fi; \ fi else # try to use the SO_NAME (if the named file exists) @@ -164,10 +153,10 @@ endif ## debug code: `make resolve-path P=` #resolve_path += && \ -# echo "$${$1_wd} $${$1}" +# printf "%s\n" "$${$1_wd} $${$1}" #resolve-path: # $(call resolve_path,P) && \ -# echo "$$P" +# printf "%s\n" "$$P" define symlink_system_library libname_$2 := $$(notdir $(call versioned_libname,$2,$3)) @@ -179,11 +168,11 @@ $$(build_private_libdir)/$$(libname_$2): $$(call resolve_path,REALPATH) && \ [ -e "$$$$REALPATH" ] && \ rm -f "$$@" && \ - echo ln -sf "$$$$REALPATH" "$$@" && \ + printf "ln -sf %s %s\n" "$$$$REALPATH" "$$@" && \ ln -sf "$$$$REALPATH" "$$@"; \ else \ if [ "$4" != "ALLOW_FAILURE" ]; then \ - echo "System library symlink failure: Unable to locate $$(libname_$2) on your system!" >&2; \ + printf "%s\n" "System library symlink failure: Unable to locate $$(libname_$2) on your system!" >&2; \ false; \ fi; \ fi @@ -295,7 +284,7 @@ $(build_private_libdir)/libLLVM.$(SHLIB_EXT): $(call resolve_path,REALPATH) && \ [ -e "$$REALPATH" ] && \ rm -f "$@" && \ - echo ln -sf "$$REALPATH" "$@" && \ + printf "%s\n" ln -sf "$$REALPATH" "$@" && \ ln -sf "$$REALPATH" "$@" ifneq ($(USE_SYSTEM_LLVM),0) ifneq ($(USE_LLVM_SHLIB),0) diff --git a/cli/Makefile b/cli/Makefile index 3cc0af1a76afd..8c73d76f5020f 100644 --- a/cli/Makefile +++ b/cli/Makefile @@ -28,8 +28,8 @@ LOADER_LDFLAGS += -Wl,--no-as-needed -lpthread -rdynamic -lc -Wl,--as-needed endif # Build list of dependent libraries that must be opened -SHIPFLAGS += -DDEP_LIBS="\"$(LOADER_BUILD_DEP_LIBS)\"" -DEBUGFLAGS += -DDEP_LIBS="\"$(LOADER_DEBUG_BUILD_DEP_LIBS)\"" +SHIPFLAGS += -DDEP_LIBS=$(call shell_escape,$(call c_escape,$(LOADER_BUILD_DEP_LIBS))) +DEBUGFLAGS += -DDEP_LIBS=$(call shell_escape,$(call c_escape,$(LOADER_DEBUG_BUILD_DEP_LIBS))) ifneq (,$(findstring MINGW,$(shell uname))) # In MSYS2, do not perform path conversion for `DEP_LIBS`. # https://www.msys2.org/wiki/Porting/#filesystem-namespaces diff --git a/src/Makefile b/src/Makefile index b49d27e05ff28..6d4a842c7a711 100644 --- a/src/Makefile +++ b/src/Makefile @@ -205,10 +205,10 @@ CODEGEN_OBJS := $(CODEGEN_SRCS:%=$(BUILDDIR)/%.o) CODEGEN_DOBJS := $(CODEGEN_SRCS:%=$(BUILDDIR)/%.dbg.obj) # Add SONAME defines so we can embed proper `dlopen()` calls. -ADDL_SHIPFLAGS := "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/sys.$(SHLIB_EXT)\"" \ - "-DJL_LIBJULIA_SONAME=\"$(LIBJULIA_PATH_REL).$(JL_MAJOR_SHLIB_EXT)\"" -ADDL_DEBUGFLAGS := "-DJL_SYSTEM_IMAGE_PATH=\"$(build_private_libdir_rel)/sys-debug.$(SHLIB_EXT)\"" \ - "-DJL_LIBJULIA_SONAME=\"$(LIBJULIA_PATH_REL)-debug.$(JL_MAJOR_SHLIB_EXT)\"" +ADDL_SHIPFLAGS := -DJL_SYSTEM_IMAGE_PATH=$(call shell_escape,$(call c_escape,$(call normalize_path,$(build_private_libdir_rel)/sys.$(SHLIB_EXT)))) \ + -DJL_LIBJULIA_SONAME=$(call shell_escape,$(call c_escape,$(LIBJULIA_PATH_REL).$(JL_MAJOR_SHLIB_EXT))) +ADDL_DEBUGFLAGS := -DJL_SYSTEM_IMAGE_PATH=$(call shell_escape,$(call c_escape,$(call normalize_path,$(build_private_libdir_rel)/sys-debug.$(SHLIB_EXT)))) \ + -DJL_LIBJULIA_SONAME=$(call shell_escape,$(call c_escape,$(LIBJULIA_PATH_REL)-debug.$(JL_MAJOR_SHLIB_EXT))) SHIPFLAGS += $(FLAGS) $(ADDL_SHIPFLAGS) DEBUGFLAGS += $(FLAGS) $(ADDL_DEBUGFLAGS) diff --git a/src/flisp/Makefile b/src/flisp/Makefile index 17292d301115b..eca1de86e588a 100644 --- a/src/flisp/Makefile +++ b/src/flisp/Makefile @@ -111,10 +111,10 @@ $(BUILDDIR)/$(EXENAME)$(EXE): $(OBJS) $(LIBFILES_release) $(BUILDDIR)/$(LIBTARGE $(BUILDDIR)/host/Makefile: mkdir -p $(BUILDDIR)/host @# add Makefiles to the build directories for convenience (pointing back to the source location of each) - @echo '# -- This file is automatically generated in julia/src/flisp/Makefile -- #' > $@ - @echo 'BUILDDIR=$(BUILDDIR)/host' >> $@ - @echo 'BUILDING_HOST_TOOLS=1' >> $@ - @echo 'include $(SRCDIR)/Makefile' >> $@ + @printf "%s\n" '# -- This file is automatically generated in julia/src/flisp/Makefile -- #' > $@ + @printf "%s\n" 'BUILDDIR=$(BUILDDIR)/host' >> $@ + @printf "%s\n" 'BUILDING_HOST_TOOLS=1' >> $@ + @printf "%s\n" 'include $(SRCDIR)/Makefile' >> $@ $(BUILDDIR)/host/$(EXENAME): $(BUILDDIR)/host/Makefile | ${BUILDDIR}/host/flisp.boot make -C $(BUILDDIR)/host $(EXENAME) diff --git a/src/support/Makefile b/src/support/Makefile index 1ee98a4eabdee..c7de154058586 100644 --- a/src/support/Makefile +++ b/src/support/Makefile @@ -48,10 +48,10 @@ $(BUILDDIR)/%.dbg.obj: $(SRCDIR)/%.S | $(BUILDDIR) $(BUILDDIR)/host/Makefile: mkdir -p $(BUILDDIR)/host @# add Makefiles to the build directories for convenience (pointing back to the source location of each) - @echo '# -- This file is automatically generated in julia/Makefile -- #' > $@ - @echo 'BUILDDIR=$(BUILDDIR)/host' >> $@ - @echo 'BUILDING_HOST_TOOLS=1' >> $@ - @echo 'include $(SRCDIR)/Makefile' >> $@ + @printf "%s\n" '# -- This file is automatically generated in julia/Makefile -- #' > $@ + @printf "%s\n" 'BUILDDIR=$(BUILDDIR)/host' >> $@ + @printf "%s\n" 'BUILDING_HOST_TOOLS=1' >> $@ + @printf "%s\n" 'include $(SRCDIR)/Makefile' >> $@ release: $(BUILDDIR)/libsupport.a debug: $(BUILDDIR)/libsupport-debug.a From d3499d04c5e8aee7cb125f206e6a18410a2911a1 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 22 Feb 2025 05:59:25 -0500 Subject: [PATCH 02/74] strip Module filename from metadata (#57499) Fix #57358 (cherry picked from commit 88aa27f482d19ae7ac0eb080c115640a0754d819) --- src/staticdata.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/staticdata.c b/src/staticdata.c index dc30ed7f64341..2be949d408756 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -794,6 +794,8 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ { jl_queue_for_serialization(s, m->name); jl_queue_for_serialization(s, m->parent); + if (jl_options.strip_metadata) + jl_queue_for_serialization(s, m->file); if (jl_options.trim) { jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->bindings), 0, 1); } else { @@ -1339,7 +1341,9 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t arraylist_push(&s->relocs_list, (void*)backref_id(s, jl_atomic_load_relaxed(&m->bindingkeyset), s->link_ids_relocs)); newm->file = NULL; arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, file))); - arraylist_push(&s->relocs_list, (void*)backref_id(s, m->file, s->link_ids_relocs)); + arraylist_push(&s->relocs_list, (void*)backref_id(s, jl_options.strip_metadata ? jl_empty_sym : m->file , s->link_ids_relocs)); + if (jl_options.strip_metadata) + newm->line = 0; newm->usings_backedges = NULL; arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings_backedges))); arraylist_push(&s->relocs_list, (void*)backref_id(s, m->usings_backedges, s->link_ids_relocs)); @@ -2593,7 +2597,7 @@ uint_t bindingkey_hash(size_t idx, jl_value_t *data); static void jl_prune_module_bindings(jl_module_t * m) JL_GC_DISABLED { - jl_svec_t * bindings = jl_atomic_load_relaxed(&m->bindings); + jl_svec_t *bindings = jl_atomic_load_relaxed(&m->bindings); size_t l = jl_svec_len(bindings), i; arraylist_t bindings_list; arraylist_new(&bindings_list, 0); From b565c2818cbf73624d7d95b5839ae7e791e7db68 Mon Sep 17 00:00:00 2001 From: PatrickHaecker <152268010+PatrickHaecker@users.noreply.github.com> Date: Sun, 23 Feb 2025 16:17:53 +0100 Subject: [PATCH 03/74] `Base.summarysize` for `Memory` with `Union` (#57508) Fixes the double accounting of the union byte array in `Base.summarysize` as described in #57506. If this is the correct fix, can it be backported to 1.11? Fix #57506 (cherry picked from commit 7b7ba33021ba1b427ee0f482999674536e7cbcef) --- base/summarysize.jl | 7 +------ test/misc.jl | 5 +++++ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/base/summarysize.jl b/base/summarysize.jl index 4f2646c7641b7..62b0ad0849778 100644 --- a/base/summarysize.jl +++ b/base/summarysize.jl @@ -149,13 +149,8 @@ function (ss::SummarySize)(obj::GenericMemory) datakey = unsafe_convert(Ptr{Cvoid}, obj) if !haskey(ss.seen, datakey) ss.seen[datakey] = true - dsize = sizeof(obj) + size += sizeof(obj) T = eltype(obj) - if isbitsunion(T) - # add 1 union selector byte for each element - dsize += length(obj) - end - size += dsize if !isempty(obj) && T !== Symbol && (!Base.allocatedinline(T) || (T isa DataType && !Base.datatype_pointerfree(T))) push!(ss.frontier_x, obj) push!(ss.frontier_i, 1) diff --git a/test/misc.jl b/test/misc.jl index bcc7ff69339a9..6c8d76fa1cd1a 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -619,6 +619,11 @@ let z = Z53061[Z53061(S53061(rand(), (rand(),rand())), 0) for _ in 1:10^4] @test abs(summarysize(z) - 640000)/640000 <= 0.01 broken = Sys.WORD_SIZE == 32 && Sys.islinux() end +# issue #57506 +let len = 100, m1 = Memory{UInt8}(1:len), m2 = Memory{Union{Nothing,UInt8}}(1:len) + @test summarysize(m2) == summarysize(m1) + len +end + ## test conversion from UTF-8 to UTF-16 (for Windows APIs) # empty arrays From d04631d62a38c9807c55651ea37149beb8adf9e2 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Tue, 25 Feb 2025 11:15:39 +0100 Subject: [PATCH 04/74] don't mutate globals when constructing `Rational` from `AbstractIrrational` (#55853) Relying on `ScopedValues`, set `BigFloat` precision without mutating the global default, while constructing `Rational` from `AbstractIrrational`. Also helps avoid reading the global defaults for the precision and rounding mode, together with #56095. What does this fix: * in the case of the `Irrational` constants defined in `MathConstants`: relevant methods have `@assume_effects :foldable` applied, which includes `:effect_free`, which requires that no globals be mutated (followup on #55886) * in the case of `AbstractIrrational` values in general, this PR prevents data races on the global `BigFloat` precision (cherry picked from commit 2a89f71fc9a00f857506279bd4987652f730063a) --- base/irrationals.jl | 34 ++++++++++++---- test/threads_exec.jl | 92 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 7 deletions(-) diff --git a/base/irrationals.jl b/base/irrationals.jl index 76222997865c0..f86b55e4faaa7 100644 --- a/base/irrationals.jl +++ b/base/irrationals.jl @@ -60,20 +60,40 @@ AbstractFloat(x::AbstractIrrational) = Float64(x)::Float64 Float16(x::AbstractIrrational) = Float16(Float32(x)::Float32) Complex{T}(x::AbstractIrrational) where {T<:Real} = Complex{T}(T(x)) -function _irrational_to_rational(::Type{T}, x::AbstractIrrational) where T<:Integer - o = precision(BigFloat) +function _irrational_to_rational_at_current_precision(::Type{T}, x::AbstractIrrational) where {T <: Integer} + bx = BigFloat(x) + r = rationalize(T, bx, tol = 0) + if abs(BigFloat(r) - bx) > eps(bx) + r + else + nothing # Error is too small, repeat with greater precision. + end +end +function _irrational_to_rational_at_precision(::Type{T}, x::AbstractIrrational, p::Int) where {T <: Integer} + f = let x = x + () -> _irrational_to_rational_at_current_precision(T, x) + end + setprecision(f, BigFloat, p) +end +function _irrational_to_rational_at_current_rounding_mode(::Type{T}, x::AbstractIrrational) where {T <: Integer} + if T <: BigInt + _throw_argument_error_irrational_to_rational_bigint() # avoid infinite loop + end p = 256 while true - setprecision(BigFloat, p) - bx = BigFloat(x) - r = rationalize(T, bx, tol=0) - if abs(BigFloat(r) - bx) > eps(bx) - setprecision(BigFloat, o) + r = _irrational_to_rational_at_precision(T, x, p) + if r isa Number return r end p += 32 end end +function _irrational_to_rational(::Type{T}, x::AbstractIrrational) where {T <: Integer} + f = let x = x + () -> _irrational_to_rational_at_current_rounding_mode(T, x) + end + setrounding(f, BigFloat, RoundNearest) +end Rational{T}(x::AbstractIrrational) where {T<:Integer} = _irrational_to_rational(T, x) _throw_argument_error_irrational_to_rational_bigint() = throw(ArgumentError("Cannot convert an AbstractIrrational to a Rational{BigInt}: use rationalize(BigInt, x) instead")) Rational{BigInt}(::AbstractIrrational) = _throw_argument_error_irrational_to_rational_bigint() diff --git a/test/threads_exec.jl b/test/threads_exec.jl index 7091893908cb3..4b8a358d05ca4 100644 --- a/test/threads_exec.jl +++ b/test/threads_exec.jl @@ -28,6 +28,61 @@ end # (expected test duration is about 18-180 seconds) Timer(t -> killjob("KILLING BY THREAD TEST WATCHDOG\n"), 1200) +module ConcurrencyUtilities + function new_task_nonsticky(f) + t = Task(f) + t.sticky = false + t + end + + """ + run_concurrently(worker, n)::Nothing + + Run `n` tasks of `worker` concurrently. Return when all workers are done. + """ + function run_concurrently(worker, n) + tasks = map(new_task_nonsticky ∘ Returns(worker), Base.OneTo(n)) + foreach(schedule, tasks) + foreach(fetch, tasks) + end + + """ + run_concurrently_in_new_task(worker, n)::Task + + Return a task that: + * is not started yet + * when started, runs `n` tasks of `worker` concurrently + * returns when all workers are done + """ + function run_concurrently_in_new_task(worker, n) + function f(t) + run_concurrently(t...) + end + new_task_nonsticky(f ∘ Returns((worker, n))) + end +end + +module AbstractIrrationalExamples + for n ∈ 0:9 + name_aa = Symbol(:aa, n) + name_ab = Symbol(:ab, n) + name_ba = Symbol(:ba, n) + name_bb = Symbol(:bb, n) + @eval begin + Base.@irrational $name_aa exp(BigFloat(2)^$n) + Base.@irrational $name_ab exp(BigFloat(2)^-$n) + Base.@irrational $name_ba exp(-(BigFloat(2)^$n)) + Base.@irrational $name_bb exp(-(BigFloat(2)^-$n)) + end + end + const examples = ( + aa0, aa1, aa2, aa3, aa4, aa5, aa6, aa7, aa8, aa9, + ab0, ab1, ab2, ab3, ab4, ab5, ab6, ab7, ab8, ab9, + ba0, ba1, ba2, ba3, ba4, ba5, ba6, ba7, ba8, ba9, + bb0, bb1, bb2, bb3, bb4, bb5, bb6, bb7, bb8, bb9, + ) +end + @testset """threads_exec.jl with JULIA_NUM_THREADS == $(ENV["JULIA_NUM_THREADS"])""" begin @test Threads.threadid() == 1 @@ -1347,6 +1402,43 @@ end end end +@testset "race on `BigFloat` precision when constructing `Rational` from `AbstractIrrational`" begin + function test_racy_rational_from_irrational(::Type{Rational{I}}, c::AbstractIrrational) where {I} + function construct() + Rational{I}(c) + end + function is_racy_rational_from_irrational() + worker_count = 10 * Threads.nthreads() + task = ConcurrencyUtilities.run_concurrently_in_new_task(construct, worker_count) + schedule(task) + ok = true + while !istaskdone(task) + for _ ∈ 1:1000000 + ok &= precision(BigFloat) === prec + end + GC.safepoint() + yield() + end + fetch(task) + ok + end + prec = precision(BigFloat) + task = ConcurrencyUtilities.new_task_nonsticky(is_racy_rational_from_irrational) + schedule(task) + ok = fetch(task)::Bool + setprecision(BigFloat, prec) + ok + end + @testset "c: $c" for c ∈ AbstractIrrationalExamples.examples + Q = Rational{Int128} + # metatest: `test_racy_rational_from_irrational` needs the constructor + # to not be constant folded away, otherwise it's not testing anything. + @test !Core.Compiler.is_foldable(Base.infer_effects(Q, Tuple{typeof(c)})) + # test for race + @test test_racy_rational_from_irrational(Q, c) + end +end + @testset "task time counters" begin @testset "enabled" begin try From 92396aa4042a619b9355a414219e2e01c2a5a24a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 26 Feb 2025 14:03:03 -0500 Subject: [PATCH 05/74] [Compiler] begin new approach to verify --trim output (#57530) Reimplement all of the trim verification support in Julia as a compiler analysis pass. Move all this verification code into the Compiler julia code, where we have much better utilities for pretty printing and better observability for analysis. (cherry picked from commit 1045bd83337d8665c88444f976614e152d3c1e79) --- Compiler/src/Compiler.jl | 9 +- Compiler/src/bootstrap.jl | 2 +- Compiler/src/typeinfer.jl | 17 +- Compiler/src/verifytrim.jl | 344 ++++++++++++++++++++++++++++++++++++ Compiler/test/testgroups | 1 + Compiler/test/verifytrim.jl | 91 ++++++++++ base/Base.jl | 7 +- base/show.jl | 1 - base/stacktraces.jl | 13 +- src/aotcompile.cpp | 36 +--- src/cgutils.cpp | 12 -- src/codegen.cpp | 291 ------------------------------ src/init.c | 3 +- src/jitlayers.h | 1 - src/julia.h | 1 - sysimage.mk | 6 +- 16 files changed, 480 insertions(+), 355 deletions(-) create mode 100644 Compiler/src/verifytrim.jl create mode 100644 Compiler/test/verifytrim.jl diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index 2c68729ee1dc2..fc2ac491f49d6 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -188,14 +188,19 @@ macro __SOURCE_FILE__() return QuoteNode(__source__.file::Symbol) end -module IRShow end +module IRShow end # relies on string and IO operations defined in Base +baremodule TrimVerifier end # relies on IRShow, so define this afterwards + function load_irshow!() if isdefined(Base, :end_base_include) # This code path is exclusively for Revise, which may want to re-run this # after bootstrap. - include(IRShow, Base.joinpath(Base.dirname(Base.String(@__SOURCE_FILE__)), "ssair/show.jl")) + Compilerdir = Base.dirname(Base.String(@__SOURCE_FILE__)) + include(IRShow, Base.joinpath(Compilerdir, "ssair/show.jl")) + include(TrimVerifier, Base.joinpath(Compilerdir, "verifytrim.jl")) else include(IRShow, "ssair/show.jl") + include(TrimVerifier, "verifytrim.jl") end end if !isdefined(Base, :end_base_include) diff --git a/Compiler/src/bootstrap.jl b/Compiler/src/bootstrap.jl index c35de48cd20f5..2671ea114e818 100644 --- a/Compiler/src/bootstrap.jl +++ b/Compiler/src/bootstrap.jl @@ -71,7 +71,7 @@ function bootstrap!() end end end - codeinfos = typeinf_ext_toplevel(methods, [world], false) + codeinfos = typeinf_ext_toplevel(methods, [world], TRIM_NO) for i = 1:2:length(codeinfos) ci = codeinfos[i]::CodeInstance src = codeinfos[i + 1]::CodeInfo diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index ddcca9a6ffaa1..2ae3242be4027 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1273,7 +1273,12 @@ function typeinf_ext_toplevel(mi::MethodInstance, world::UInt, source_mode::UInt end # This is a bridge for the C code calling `jl_typeinf_func()` on set of Method matches -function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim::Bool) +# The trim_mode can be any of: +const TRIM_NO = 0 +const TRIM_SAFE = 1 +const TRIM_UNSAFE = 2 +const TRIM_UNSAFE_WARN = 3 +function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_mode::Int) inspected = IdSet{CodeInstance}() tocompile = Vector{CodeInstance}() codeinfos = [] @@ -1321,7 +1326,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: src = codeinfo_for_const(interp, mi, callee.rettype_const) elseif haskey(interp.codegen, callee) src = interp.codegen[callee] - elseif isa(def, Method) && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 && !trim + elseif isa(def, Method) && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 && trim_mode == TRIM_NO src = retrieve_code_info(mi, get_inference_world(interp)) else # TODO: typeinf_code could return something with different edges/ages/owner/abi (needing an update to callee), which we don't handle here @@ -1336,15 +1341,19 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim:: end push!(codeinfos, callee) push!(codeinfos, src) - elseif trim - println("warning: failed to get code for ", mi) end end latest = false end + if trim_mode != TRIM_NO && trim_mode != TRIM_UNSAFE + verify_typeinf_trim(codeinfos, trim_mode == TRIM_UNSAFE_WARN) + end return codeinfos end +verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) = (msg = "--trim verifier not defined"; onlywarn ? println(io, msg) : error(msg)) +verify_typeinf_trim(codeinfos::Vector{Any}, onlywarn::Bool) = invokelatest(verify_typeinf_trim, stdout, codeinfos, onlywarn) + function return_type(@nospecialize(f), t::DataType) # this method has a special tfunc world = tls_world_age() args = Any[_return_type, NativeInterpreter(world), Tuple{Core.Typeof(f), t.parameters...}] diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl new file mode 100644 index 0000000000000..7706d89483cbf --- /dev/null +++ b/Compiler/src/verifytrim.jl @@ -0,0 +1,344 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +import ..Compiler: verify_typeinf_trim + +using ..Compiler: + # operators + !, !=, !==, +, :, <, <=, ==, =>, >, >=, ∈, ∉, + # types + Array, Builtin, Callable, Cint, CodeInfo, CodeInstance, Csize_t, Exception, + GenericMemory, GlobalRef, IdDict, IdSet, IntrinsicFunction, Method, MethodInstance, + NamedTuple, Pair, PhiCNode, PhiNode, PiNode, QuoteNode, SSAValue, SimpleVector, String, + Tuple, VarState, Vector, + # functions + argextype, empty!, error, get, get_ci_mi, get_world_counter, getindex, getproperty, + hasintersect, haskey, in, isdispatchelem, isempty, isexpr, iterate, length, map!, max, + pop!, popfirst!, push!, pushfirst!, reinterpret, reverse!, reverse, setindex!, + setproperty!, similar, singleton_type, sptypes_from_meth_instance, + unsafe_pointer_to_objref, widenconst, + # misc + @nospecialize, C_NULL +using ..IRShow: LineInfoNode, print, show, println, append_scopes!, IOContext, IO, normalize_method_name +using ..Base: Base, sourceinfo_slotnames +using ..Base.StackTraces: StackFrame + +## declarations ## + +struct CallMissing <: Exception + codeinst::CodeInstance + codeinfo::CodeInfo + sptypes::Vector{VarState} + stmtidx::Int + desc::String +end + +struct CCallableMissing <: Exception + rt + sig + desc +end + +const ParentMap = IdDict{CodeInstance,Tuple{CodeInstance,Int}} +const ErrorList = Vector{Pair{Bool,Any}} # severity => exception + +const runtime_functions = Symbol[ + # a denylist of any runtime functions which someone might ccall which can call jl_apply or access reflection state + # which might not be captured by the trim output + :jl_apply, +] + +## code for pretty printing ## + +# wrap a statement in a typeassert for printing clarity, unless that info seems already obvious +function mapssavaluetypes(codeinfo::CodeInfo, sptypes::Vector{VarState}, stmt) + @nospecialize stmt + newstmt = mapssavalues(codeinfo, sptypes, stmt) + typ = widenconst(argextype(stmt, codeinfo, sptypes)) + if newstmt isa Expr + if newstmt.head ∈ (:quote, :inert) + return newstmt + end + elseif newstmt isa GlobalRef && isdispatchelem(typ) + return newstmt + elseif newstmt isa Union{Int, UInt8, UInt16, UInt32, UInt64, Float16, Float32, Float64, String, QuoteNode} + return newstmt + elseif newstmt isa Callable + return newstmt + end + return Expr(:(::), newstmt, typ) +end + +# map the ssavalues in a (value-producing) statement to the expression they came from, summarizing some things to avoid excess printing +function mapssavalues(codeinfo::CodeInfo, sptypes::Vector{VarState}, stmt) + @nospecialize stmt + if stmt isa SSAValue + return mapssavalues(codeinfo, sptypes, codeinfo.code[stmt.id]) + elseif stmt isa PiNode + return mapssavalues(codeinfo, sptypes, stmt.val) + elseif stmt isa Expr + stmt.head ∈ (:quote, :inert) && return stmt + newstmt = Expr(stmt.head) + if stmt.head === :foreigncall + return Expr(:call, :ccall, mapssavalues(codeinfo, sptypes, stmt.args[1])) + elseif stmt.head ∉ (:new, :method, :toplevel, :thunk) + newstmt.args = map!(similar(stmt.args), stmt.args) do arg + @nospecialize arg + return mapssavaluetypes(codeinfo, sptypes, arg) + end + if newstmt.head === :invoke + # why is the fancy printing for this not in show_unquoted? + popfirst!(newstmt.args) + newstmt.head = :call + end + end + return newstmt + elseif stmt isa PhiNode + return PhiNode() + elseif stmt isa PhiCNode + return PhiNode() + end + return stmt +end + +function verify_print_stmt(io::IOContext{IO}, codeinfo::CodeInfo, sptypes::Vector{VarState}, stmtidx::Int) + if codeinfo.slotnames !== nothing + io = IOContext(io, :SOURCE_SLOTNAMES => sourceinfo_slotnames(codeinfo)) + end + print(io, mapssavaluetypes(codeinfo, sptypes, SSAValue(stmtidx))) +end + +function verify_print_error(io::IOContext{IO}, desc::CallMissing, parents::ParentMap) + (; codeinst, codeinfo, sptypes, stmtidx, desc) = desc + frames = verify_create_stackframes(codeinst, stmtidx, parents) + print(io, desc, " from ") + verify_print_stmt(io, codeinfo, sptypes, stmtidx) + Base.show_backtrace(io, frames) + print(io, "\n\n") + nothing +end + +function verify_print_error(io::IOContext{IO}, desc::CCallableMissing, parents::ParentMap) + print(io, desc.desc, " for ", desc.sig, " => ", desc.rt, "\n\n") + nothing +end + +function verify_create_stackframes(codeinst::CodeInstance, stmtidx::Int, parents::ParentMap) + scopes = LineInfoNode[] + frames = StackFrame[] + parent = (codeinst, stmtidx) + while parent !== nothing + codeinst, stmtidx = parent + di = codeinst.debuginfo + append_scopes!(scopes, stmtidx, di, :var"unknown scope") + for i in reverse(1:length(scopes)) + lno = scopes[i] + inlined = i != 1 + def = lno.method + def isa Union{Method,Core.CodeInstance,MethodInstance} || (def = nothing) + sf = StackFrame(normalize_method_name(lno.method), lno.file, lno.line, def, false, inlined, 0) + push!(frames, sf) + end + empty!(scopes) + parent = get(parents, codeinst, nothing) + end + return frames +end + +## code for analysis ## + +function may_dispatch(@nospecialize ftyp) + if ftyp <: IntrinsicFunction + return true + elseif ftyp <: Builtin + # other builtins (including the IntrinsicFunctions) are good + return Core._apply isa ftyp || + Core._apply_iterate isa ftyp || + Core._apply_pure isa ftyp || + Core._call_in_world isa ftyp || + Core._call_in_world_total isa ftyp || + Core._call_latest isa ftyp || + Core.invoke isa ftyp || + Core.finalizer isa ftyp || + Core.modifyfield! isa ftyp || + Core.modifyglobal! isa ftyp || + Core.memoryrefmodify! isa ftyp + else + return true + end +end + +function verify_codeinstance!(codeinst::CodeInstance, codeinfo::CodeInfo, inspected::IdSet{CodeInstance}, caches::IdDict{MethodInstance,CodeInstance}, parents::ParentMap, errors::ErrorList) + mi = get_ci_mi(codeinst) + sptypes = sptypes_from_meth_instance(mi) + src = codeinfo.code + for i = 1:length(src) + stmt = src[i] + isexpr(stmt, :(=)) && (stmt = stmt.args[2]) + error = "" + warn = false + if isexpr(stmt, :invoke) || isexpr(stmt, :invoke_modify) + error = "unresolved invoke" + edge = stmt.args[1] + if edge isa CodeInstance + haskey(parents, edge) || (parents[edge] = (codeinst, i)) + edge in inspected && continue + end + # TODO: check for calls to Base.atexit? + elseif isexpr(stmt, :call) + error = "unresolved call" + farg = stmt.args[1] + ftyp = widenconst(argextype(farg, codeinfo, sptypes)) + if ftyp <: IntrinsicFunction + #TODO: detect if f !== Core.Intrinsics.atomic_pointermodify (see statement_cost), otherwise error + continue + elseif ftyp <: Builtin + if !may_dispatch(ftyp) + continue + end + if Core._apply_iterate isa ftyp + if length(stmt.args) >= 3 + # args[1] is _apply_iterate object + # args[2] is invoke object + farg = stmt.args[3] + ftyp = widenconst(argextype(farg, codeinfo, sptypes)) + if may_dispatch(ftyp) + error = "unresolved call to function" + else + for i in 4:length(stmt.args) + atyp = widenconst(argextype(stmt.args[i], codeinfo, sptypes)) + if !(atyp <: Union{SimpleVector, GenericMemory, Array, Tuple, NamedTuple}) + error = "unresolved argument to call" + break + end + end + end + end + elseif Core.finalizer isa ftyp + if length(stmt.args) == 3 + # TODO: check that calling `args[1](args[2])` is defined before warning + error = "unresolved finalizer registered" + warn = true + end + else + error = "unresolved call to builtin" + end + end + extyp = argextype(SSAValue(i), codeinfo, sptypes) + if extyp === Union{} + warn = true # downgrade must-throw calls to be only a warning + end + elseif isexpr(stmt, :cfunction) + error = "unresolved cfunction" + #TODO: parse the cfunction expression to check the target is defined + warn = true + elseif isexpr(stmt, :foreigncall) + foreigncall = stmt.args[1] + if foreigncall isa QuoteNode + if foreigncall.value in runtime_functions + error = "disallowed ccall into a runtime function" + end + end + elseif isexpr(stmt, :new_opaque_closure) + error = "unresolved opaque closure" + # TODO: check that this opaque closure has a valid signature for possible codegen and code defined for it + warn = true + end + if !isempty(error) + push!(errors, warn => CallMissing(codeinst, codeinfo, sptypes, i, error)) + end + end +end + +## entry-point ## + +function get_verify_typeinf_trim(codeinfos::Vector{Any}) + this_world = get_world_counter() + inspected = IdSet{CodeInstance}() + caches = IdDict{MethodInstance,CodeInstance}() + errors = ErrorList() + parents = ParentMap() + for i = 1:2:length(codeinfos) + item = codeinfos[i] + if item isa CodeInstance + push!(inspected, item) + if item.owner === nothing && item.min_world <= this_world <= item.max_world + mi = get_ci_mi(item) + if mi === item.def + caches[mi] = item + end + end + end + end + for i = 1:2:length(codeinfos) + item = codeinfos[i] + if item isa CodeInstance + src = codeinfos[i + 1]::CodeInfo + verify_codeinstance!(item, src, inspected, caches, parents, errors) + else + rt = item::Type + sig = codeinfos[i + 1]::Type + ptr = ccall(:jl_get_specialization1, + #= MethodInstance =# Ptr{Cvoid}, (Any, Csize_t, Cint), + sig, this_world, #= mt_cache =# 0) + asrt = Any + valid = if ptr !== C_NULL + mi = unsafe_pointer_to_objref(ptr)::MethodInstance + ci = get(caches, mi, nothing) + if ci isa CodeInstance + # TODO: should we find a way to indicate to the user that this gets called via ccallable? + # parent[ci] = something + asrt = ci.rettype + ci in inspected + else + false + end + else + false + end + if !valid + warn = false + push!(errors, warn => CCallableMissing(rt, sig, "unresolved ccallable")) + elseif !(asrt <: rt) + warn = hasintersect(asrt, rt) + push!(errors, warn => CCallableMissing(asrt, sig, "ccallable declared return type does not match inference")) + end + end + end + return (errors, parents) +end + +# It is unclear if this file belongs in Compiler itself, or should instead be a codegen +# driver / verifier implemented by juliac-buildscript.jl for the purpose of extensibility. +# For now, it is part of Base.Compiler, but executed with invokelatest so that packages +# could provide hooks to change, customize, or tweak its behavior and heuristics. +Base.delete_method(Base.which(verify_typeinf_trim, (IO, Vector{Any}, Bool)),) +function verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) + errors, parents = get_verify_typeinf_trim(codeinfos) + + # count up how many messages we printed, of each severity + counts = [0, 0] # errors, warnings + io = IOContext{IO}(io) + # print all errors afterwards, when the parents map is fully constructed + for desc in errors + warn, desc = desc + severity = warn ? 2 : 1 + no = (counts[severity] += 1) + print(io, warn ? "Verifier warning #" : "Verifier error #", no, ": ") + # TODO: should we coalesce any of these stacktraces to minimize spew? + verify_print_error(io, desc, parents) + end + + let severity = 0 + if counts[2] > 0 + print("Trim verify finished with ", counts[2], counts[2] == 1 ? " warning.\n\n" : " warnings.\n\n") + severity = 2 + end + if counts[1] > 0 + print("Trim verify finished with ", counts[1], counts[1] == 1 ? " error.\n\n" : " errors.\n\n") + severity = 1 + end + # messages classified as errors are fatal, warnings are not + 0 < severity <= 1 && !onlywarn && error("verify_typeinf_trim failed") + end + nothing +end diff --git a/Compiler/test/testgroups b/Compiler/test/testgroups index d17735a52a025..4656448016cd3 100644 --- a/Compiler/test/testgroups +++ b/Compiler/test/testgroups @@ -16,3 +16,4 @@ tarjan validation special_loading abioverride +verifytrim diff --git a/Compiler/test/verifytrim.jl b/Compiler/test/verifytrim.jl new file mode 100644 index 0000000000000..a03804a94cb62 --- /dev/null +++ b/Compiler/test/verifytrim.jl @@ -0,0 +1,91 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +include("setup_Compiler.jl") + +# revise: Core.include(Compiler.TrimVerifier, joinpath(@__DIR__, "../src/verifytrim.jl")) + +using Test +using .Compiler: typeinf_ext_toplevel, TrimVerifier, TRIM_SAFE, TRIM_UNSAFE +using .TrimVerifier: get_verify_typeinf_trim, verify_print_error, CallMissing, CCallableMissing + +function sprint(f, args...) + return Base.sprint((io, f, args...) -> f(IOContext{IO}(io), args...), f, args...) +end + +let infos = Any[] + errors, parents = get_verify_typeinf_trim(infos) + @test isempty(errors) + @test isempty(parents) +end + +# use TRIM_UNSAFE to bypass verifier inside typeinf_ext_toplevel +let infos = typeinf_ext_toplevel(Any[Core.svec(Base.SecretBuffer, Tuple{Type{Base.SecretBuffer}})], [Base.get_world_counter()], TRIM_UNSAFE) + @test_broken length(infos) > 4 + errors, parents = get_verify_typeinf_trim(infos) + @test_broken isempty(errors) # missing cfunction + #resize!(infos, 4) + #errors, = get_verify_typeinf_trim(infos) + + desc = only(errors) + @test desc.first + desc = desc.second + @test desc isa CallMissing + @test occursin("finalizer", desc.desc) + repr = sprint(verify_print_error, desc, parents) + @test occursin( + r"""^unresolved finalizer registered from \(Core.finalizer\)\(Base.final_shred!, %new\(\)::Base.SecretBuffer\)::Nothing + Stacktrace: + \[1\] finalizer\(f::typeof\(Base.final_shred!\), o::Base.SecretBuffer\) + @ Base gcutils.jl:(\d+) \[inlined\] + \[2\] Base.SecretBuffer\(; sizehint::Int\d\d\) + @ Base secretbuffer.jl:(\d+) \[inlined\] + \[3\] Base.SecretBuffer\(\) + @ Base secretbuffer.jl:(\d+) + + $""", repr) + + resize!(infos, 2) + @test infos[1] isa Type && infos[2] isa Type + errors, parents = get_verify_typeinf_trim(infos) + desc = only(errors) + @test !desc.first + desc = desc.second + @test desc isa CCallableMissing + @test desc.rt == Base.SecretBuffer + @test desc.sig == Tuple{Type{Base.SecretBuffer}} + @test occursin("unresolved ccallable", desc.desc) + repr = sprint(verify_print_error, desc, parents) + @test repr == "unresolved ccallable for Tuple{Type{Base.SecretBuffer}} => Base.SecretBuffer\n\n" +end + +let infos = typeinf_ext_toplevel(Any[Core.svec(Float64, Tuple{typeof(+), Int32, Int64})], [Base.get_world_counter()], TRIM_UNSAFE) + errors, parents = get_verify_typeinf_trim(infos) + desc = only(errors) + @test !desc.first + desc = desc.second + @test desc isa CCallableMissing + @test desc.rt == Int64 + @test desc.sig == Tuple{typeof(+), Int32, Int64} + @test occursin("ccallable declared return type", desc.desc) + repr = sprint(verify_print_error, desc, parents) + @test repr == "ccallable declared return type does not match inference for Tuple{typeof(+), Int32, Int64} => Int64\n\n" +end + +let infos = typeinf_ext_toplevel(Any[Core.svec(Int64, Tuple{typeof(ifelse), Bool, Int64, UInt64})], [Base.get_world_counter()], TRIM_UNSAFE) + errors, parents = get_verify_typeinf_trim(infos) + desc = only(errors) + @test desc.first + desc = desc.second + @test desc isa CCallableMissing + @test occursin("ccallable declared return type", desc.desc) + repr = sprint(verify_print_error, desc, parents) + @test repr == "ccallable declared return type does not match inference for Tuple{typeof(ifelse), Bool, Int64, UInt64} => Union{Int64, UInt64}\n\n" +end + +let infos = typeinf_ext_toplevel(Any[Core.svec(Union{Int64,UInt64}, Tuple{typeof(ifelse), Bool, Int64, UInt64})], [Base.get_world_counter()], TRIM_SAFE) + errors, parents = get_verify_typeinf_trim(infos) + @test isempty(errors) + infos = typeinf_ext_toplevel(Any[Core.svec(Real, Tuple{typeof(ifelse), Bool, Int64, UInt64})], [Base.get_world_counter()], TRIM_SAFE) + errors, parents = get_verify_typeinf_trim(infos) + @test isempty(errors) +end diff --git a/base/Base.jl b/base/Base.jl index fde41a5f19f37..acdb54b93077b 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -107,6 +107,9 @@ include("strings/strings.jl") include("regex.jl") include("parse.jl") include("shell.jl") +const IRShow = Compiler.IRShow # an alias for compatibility +include("stacktraces.jl") +using .StackTraces include("show.jl") include("arrayshow.jl") include("methodshow.jl") @@ -243,10 +246,6 @@ include("irrationals.jl") include("mathconstants.jl") using .MathConstants: ℯ, π, pi -# Stack frames and traces -include("stacktraces.jl") -using .StackTraces - # experimental API's include("experimental.jl") diff --git a/base/show.jl b/base/show.jl index 6316f2811a34c..0cc16b11cb298 100644 --- a/base/show.jl +++ b/base/show.jl @@ -2834,7 +2834,6 @@ function show(io::IO, vm::Core.TypeofVararg) end Compiler.load_irshow!() -const IRShow = Compiler.IRShow # an alias for compatibility function show(io::IO, src::CodeInfo; debuginfo::Symbol=:source) # Fix slot names and types in function body diff --git a/base/stacktraces.jl b/base/stacktraces.jl index 01e8a3cf62e72..806c9468efed4 100644 --- a/base/stacktraces.jl +++ b/base/stacktraces.jl @@ -7,8 +7,9 @@ module StackTraces import Base: hash, ==, show -import Core: CodeInfo, MethodInstance, CodeInstance -using Base.IRShow: normalize_method_name, append_scopes!, LineInfoNode + +using Core: CodeInfo, MethodInstance, CodeInstance +using Base.IRShow export StackTrace, StackFrame, stacktrace @@ -112,7 +113,7 @@ Base.@constprop :none function lookup(pointer::Ptr{Cvoid}) res = Vector{StackFrame}(undef, length(infos)) for i in 1:length(infos) info = infos[i]::Core.SimpleVector - @assert(length(info) == 6) + @assert length(info) == 6 "corrupt return from jl_lookup_code_address" func = info[1]::Symbol file = info[2]::Symbol linenum = info[3]::Int @@ -158,8 +159,8 @@ function lookup(ip::Base.InterpreterIP) end def = (code isa CodeInfo ? StackTraces : code) # Module just used as a token for top-level code pc::Int = max(ip.stmt + 1, 0) # n.b. ip.stmt is 0-indexed - scopes = LineInfoNode[] - append_scopes!(scopes, pc, codeinfo.debuginfo, def) + scopes = IRShow.LineInfoNode[] + IRShow.append_scopes!(scopes, pc, codeinfo.debuginfo, def) if isempty(scopes) return [StackFrame(func, file, line, code, false, false, 0)] end @@ -171,7 +172,7 @@ function lookup(ip::Base.InterpreterIP) else def = codeinfo end - sf = StackFrame(normalize_method_name(lno.method), lno.file, lno.line, def, false, inlined, 0) + sf = StackFrame(IRShow.normalize_method_name(lno.method), lno.file, lno.line, def, false, inlined, 0) inlined = true return sf end diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index d91da9c64cda9..9fab5b11d97c4 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -428,12 +428,6 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root preal_specsig = true; } } - else if (params.params->trim) { - jl_safe_printf("warning: no code provided for function "); - jl_(codeinst->def); - if (params.params->trim) - abort(); - } } // patch up the prototype we emitted earlier Module *mod = proto.decl->getParent(); @@ -447,11 +441,6 @@ static void resolve_workqueue(jl_codegen_params_t ¶ms, egal_set &method_root preal_decl = mod->getNamedValue(gf_thunk_name)->getName(); } if (preal_decl.empty()) { - if (invokeName.empty() && params.params->trim) { - jl_safe_printf("warning: bailed out to invoke when compiling: "); - jl_(codeinst->def); - abort(); - } pinvoke = emit_tojlinvoke(codeinst, invokeName, mod, params); if (!proto.specsig) proto.decl->replaceAllUsesWith(pinvoke); @@ -638,9 +627,9 @@ static void generate_cfunc_thunks(jl_codegen_params_t ¶ms, jl_compiled_funct // takes the running content that has collected in the shadow module and dump it to disk // this builds the object file portion of the sysimage files for fast startup -// `_external_linkage` create linkages between pkgimages. +// `external_linkage` create linkages between pkgimages. extern "C" JL_DLLEXPORT_CODEGEN -void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, int _trim, int _external_linkage, size_t world) +void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, int trim, int external_linkage, size_t world) { JL_TIMING(INFERENCE, INFERENCE); auto ct = jl_current_task; @@ -653,10 +642,9 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm compiler_start_time = jl_hrtime(); jl_cgparams_t cgparams = jl_default_cgparams; - cgparams.trim = _trim ? 1 : 0; size_t compile_for[] = { jl_typeinf_world, world }; int compiler_world = 1; - if (_trim || compile_for[0] == 0) + if (trim || compile_for[0] == 0) compiler_world = 0; jl_value_t **fargs; JL_GC_PUSHARGS(fargs, 4); @@ -673,7 +661,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm fargs[2] = (jl_value_t*)worlds; jl_array_data(worlds, size_t)[0] = jl_typeinf_world; jl_array_data(worlds, size_t)[compiler_world] = world; // might overwrite previous - fargs[3] = _trim ? jl_true : jl_false; + fargs[3] = jl_box_long(trim); size_t last_age = ct->world_age; ct->world_age = jl_typeinf_world; codeinfos = (jl_array_t*)jl_apply(fargs, 4); @@ -685,7 +673,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm jl_error("inference not available for generating compiled output"); } fargs[0] = (jl_value_t*)codeinfos; - void *data = jl_emit_native(codeinfos, llvmmod, &cgparams, _external_linkage); + void *data = jl_emit_native(codeinfos, llvmmod, &cgparams, external_linkage); // move everything inside, now that we've merged everything // (before adding the exported headers) @@ -729,7 +717,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm // also be used be extern consumers like GPUCompiler.jl to obtain a module containing // all reachable & inferrrable functions. extern "C" JL_DLLEXPORT_CODEGEN -void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _external_linkage) +void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int external_linkage) { JL_TIMING(NATIVE_AOT, NATIVE_Create); ++CreateNativeCalls; @@ -737,7 +725,6 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm if (cgparams == NULL) cgparams = &jl_default_cgparams; jl_native_code_desc_t *data = new jl_native_code_desc_t; - jl_method_instance_t *mi = NULL; orc::ThreadSafeContext ctx; orc::ThreadSafeModule backing; if (!llvmmod) { @@ -757,7 +744,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm params.getContext().setDiscardValueNames(true); params.params = cgparams; assert(params.imaging_mode); // `_imaging_mode` controls if broken features like code-coverage are disabled - params.external_linkage = _external_linkage; + params.external_linkage = external_linkage; params.temporary_roots = jl_alloc_array_1d(jl_array_any_type, 0); JL_GC_PUSH3(¶ms.temporary_roots, &method_roots.list, &method_roots.keyset); jl_compiled_functions_t compiled_functions; @@ -773,7 +760,7 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm assert(jl_is_code_info(src)); if (compiled_functions.count(codeinst)) continue; // skip any duplicates that accidentally made there way in here (or make this an error?) - if (_external_linkage) { + if (external_linkage) { uint8_t specsigflags; jl_callptr_t invoke; void *fptr; @@ -795,13 +782,6 @@ void *jl_emit_native_impl(jl_array_t *codeinfos, LLVMOrcThreadSafeModuleRef llvm record_method_roots(method_roots, jl_get_ci_mi(codeinst)); if (result_m) compiled_functions[codeinst] = {std::move(result_m), std::move(decls)}; - else if (params.params->trim) { - // if we're building a small image, we need to compile everything - // to ensure that we have all the information we need. - jl_safe_printf("codegen failed to compile code root "); - jl_(mi); - abort(); - } } else { jl_value_t *sig = jl_array_ptr_ref(codeinfos, ++i); diff --git a/src/cgutils.cpp b/src/cgutils.cpp index eeba59aba3aed..56c5dd48e649f 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -2281,12 +2281,6 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, ret = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); } else { - if (trim_may_error(ctx.params->trim)) { - // if we know the return type, we can assume the result is of that type - errs() << "ERROR: Dynamic call to setfield/modifyfield\n"; - errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; - print_stacktrace(ctx, ctx.params->trim); - } Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); ret = mark_julia_type(ctx, callval, true, jl_any_type); } @@ -4020,12 +4014,6 @@ static jl_cgval_t union_store(jl_codectx_t &ctx, rhs = emit_invoke(ctx, *modifyop, argv, 3, (jl_value_t*)jl_any_type); } else { - if (trim_may_error(ctx.params->trim)) { - // if we know the return type, we can assume the result is of that type - errs() << "ERROR: Dynamic call to setfield/modifyfield\n"; - errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; - print_stacktrace(ctx, ctx.params->trim); - } Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, 3, julia_call); rhs = mark_julia_type(ctx, callval, true, jl_any_type); } diff --git a/src/codegen.cpp b/src/codegen.cpp index b2373ec896cc5..17d005e09200e 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1633,17 +1633,6 @@ static const auto &builtin_func_map() { return *builtins; } -static const auto &may_dispatch_builtins() { - static auto builtins = new DenseSet( - {jl_f__apply_iterate_addr, - jl_f__apply_pure_addr, - jl_f__call_in_world_addr, - jl_f__call_in_world_total_addr, - jl_f__call_latest_addr, - }); - return *builtins; -} - static const auto jl_new_opaque_closure_jlcall_func = new JuliaFunction<>{XSTR(jl_new_opaque_closure_jlcall), get_func_sig, get_func_attrs}; static _Atomic(uint64_t) globalUniqueGeneratedNames{1}; @@ -2119,182 +2108,6 @@ static Value *literal_pointer_val(jl_codectx_t &ctx, jl_value_t *p); static unsigned julia_alignment(jl_value_t *jt); static void recombine_value(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dst, jl_aliasinfo_t const &dst_ai, Align alignment, bool isVolatile); -static void print_stack_crumbs(jl_codectx_t &ctx) -{ - errs() << "\n"; - errs() << "Stacktrace:\n"; - jl_method_instance_t *caller = ctx.linfo; - jl_((jl_value_t*)caller); - errs() << "In " << ctx.file << ":" << ctx.line << "\n"; - while (true) { - auto it = ctx.emission_context.enqueuers.find(caller); - if (it != ctx.emission_context.enqueuers.end()) { - caller = std::get(it->second); - } else { - break; - } - if (caller) { - if (jl_is_method_instance(caller)) { - for (auto it2 = std::get(it->second).begin(); it2 != (std::prev(std::get(it->second).end())); ++it2) { - auto frame = *it2; - errs() << std::get<0>(frame) << " \n"; - errs() << "In " << std::get<1>(frame) << ":" << std::get(frame) << "\n"; - } - auto &frame = std::get(it->second).front(); - jl_((jl_value_t*)caller); - errs() << "In " << std::get<1>(frame) << ":" << std::get(frame) << "\n"; - } - } - else - break; - } - abort(); -} - -static jl_value_t *StackFrame( - jl_value_t *linfo, - std::string fn_name, - std::string filepath, - int32_t lineno, - jl_value_t *inlined) -{ - jl_value_t *StackFrame = jl_get_global(jl_base_module, jl_symbol("StackFrame")); - assert(StackFrame != nullptr); - - jl_value_t *args[7] = { - /* func */ (jl_value_t *)jl_symbol(fn_name.c_str()), - /* line */ (jl_value_t *)jl_symbol(filepath.c_str()), - /* line */ jl_box_int32(lineno), - /* linfo */ (jl_value_t *)linfo, - /* from_c */ jl_false, - /* inlined */ inlined, - /* pointer */ jl_box_uint64(0) - }; - - jl_value_t *frame = nullptr; - JL_TRY { - frame = jl_apply_generic(StackFrame, args, 7); - } JL_CATCH { - jl_safe_printf("Error creating stack frame\n"); - } - return frame; -} - -static void push_frames(jl_codectx_t &ctx, jl_method_instance_t *caller, jl_method_instance_t *callee) -{ - CallFrames frames; - auto it = ctx.emission_context.enqueuers.find(callee); - if (it != ctx.emission_context.enqueuers.end()) - return; - auto DL = ctx.builder.getCurrentDebugLocation(); - if (caller == nullptr || !DL) { // Used in various places - frames.push_back({ctx.funcName, "", 0}); - ctx.emission_context.enqueuers.insert({callee, {caller, std::move(frames)}}); - return; - } - auto filename = std::string(DL->getFilename()); - auto line = DL->getLine(); - auto fname = std::string(DL->getScope()->getSubprogram()->getName()); - frames.push_back({fname, filename, line}); - auto DI = DL.getInlinedAt(); - while (DI) { - auto filename = std::string(DI->getFilename()); - auto line = DI->getLine(); - auto fname = std::string(DI->getScope()->getSubprogram()->getName()); - frames.push_back({fname, filename, line}); - DI = DI->getInlinedAt(); - } - ctx.emission_context.enqueuers.insert({callee, {caller, std::move(frames)}}); -} - -static jl_array_t* build_stack_crumbs(jl_codectx_t &ctx) JL_NOTSAFEPOINT -{ - static intptr_t counter = 5; - jl_method_instance_t *caller = (jl_method_instance_t*)counter; //nothing serves as a sentinel for the bottom for the stack - push_frames(ctx, ctx.linfo, (jl_method_instance_t*)caller); - counter++; - jl_array_t *out = jl_alloc_array_1d(jl_array_any_type, 0); - JL_GC_PUSH1(&out); - while (true) { - auto it = ctx.emission_context.enqueuers.find(caller); - if (it != ctx.emission_context.enqueuers.end()) { - caller = std::get(it->second); - } else { - break; - } - if (caller) { - - // assert(ctx.emission_context.enqueuers.count(caller) == 1); - // Each enqueuer should only be enqueued at least once and only once. Check why this assert is triggering - // This isn't a fatal error, just means that we may get a wrong backtrace - if (jl_is_method_instance(caller)) { - //TODO: Use a subrange when C++20 is a thing - for (auto it2 = std::get(it->second).begin(); it2 != (std::prev(std::get(it->second).end())); ++it2) { - auto frame = *it2; - jl_value_t *stackframe = StackFrame(jl_nothing, std::get<0>(frame), std::get<1>(frame), std::get(frame), jl_true); - if (stackframe == nullptr) - print_stack_crumbs(ctx); - jl_array_ptr_1d_push(out, stackframe); - } - auto &frame = std::get(it->second).back(); - jl_value_t *stackframe = StackFrame((jl_value_t *)caller, std::get<0>(frame), std::get<1>(frame), std::get(frame), jl_false); - if (stackframe == nullptr) - print_stack_crumbs(ctx); - jl_array_ptr_1d_push(out, stackframe); - } - } - else - break; - } - JL_GC_POP(); - return out; -} - -static void print_stacktrace(jl_codectx_t &ctx, int trim) -{ - jl_task_t *ct = jl_get_current_task(); - assert(ct); - - // Temporarily operate in the current age - size_t last_age = ct->world_age; - ct->world_age = jl_get_world_counter(); - jl_array_t* bt = build_stack_crumbs(ctx); - JL_GC_PUSH1(&bt); - - // Call `reinit_stdio` to get TTY IO objects (w/ color) - jl_value_t *reinit_stdio = jl_get_global(jl_base_module, jl_symbol("_reinit_stdio")); - assert(reinit_stdio); - jl_apply_generic(reinit_stdio, nullptr, 0); - - // Show the backtrace - jl_value_t *show_backtrace = jl_get_global(jl_base_module, jl_symbol("show_backtrace")); - jl_value_t *base_stderr = jl_get_global(jl_base_module, jl_symbol("stderr")); - assert(show_backtrace && base_stderr); - - JL_TRY { - jl_value_t *args[2] = { base_stderr, (jl_value_t *)bt }; - jl_apply_generic(show_backtrace, args, 2); - } JL_CATCH { - jl_printf(JL_STDERR,"Error showing backtrace\n"); - print_stack_crumbs(ctx); - } - - jl_printf(JL_STDERR, "\n\n"); - JL_GC_POP(); - ct->world_age = last_age; - - if (trim == JL_TRIM_SAFE) { - jl_printf(JL_STDERR,"Aborting compilation due to finding a dynamic dispatch"); - exit(1); - } - return; -} - -static int trim_may_error(int trim) -{ - return (trim == JL_TRIM_SAFE) || (trim == JL_TRIM_UNSAFE_WARN); -} - static GlobalVariable *prepare_global_in(Module *M, JuliaVariable *G) { return G->realize(M); @@ -4326,12 +4139,6 @@ static bool emit_builtin_call(jl_codectx_t &ctx, jl_cgval_t *ret, jl_value_t *f, Value *theArgs = emit_ptrgep(ctx, ctx.argArray, ctx.nReqArgs * sizeof(jl_value_t*)); Value *r = ctx.builder.CreateCall(prepare_call(jlapplygeneric_func), { theF, theArgs, nva }); *ret = mark_julia_type(ctx, r, true, jl_any_type); - if (trim_may_error(ctx.params->trim)) { - // if we know the return type, we can assume the result is of that type - errs() << "ERROR: Dynamic call to Core._apply_iterate detected\n"; - errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; - print_stacktrace(ctx, ctx.params->trim); - } return true; } } @@ -5485,8 +5292,6 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR bool specsig, needsparams; std::tie(specsig, needsparams) = uses_specsig(get_ci_abi(codeinst), mi, codeinst->rettype, ctx.params->prefer_specsig); if (needsparams) { - if (trim_may_error(ctx.params->trim)) - push_frames(ctx, ctx.linfo, mi); Value *r = emit_jlcall(ctx, jlinvoke_func, track_pjlvalue(ctx, literal_pointer_val(ctx, (jl_value_t*)mi)), argv, nargs, julia_call2); result = mark_julia_type(ctx, r, true, rt); } @@ -5541,8 +5346,6 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); ctx.call_targets[codeinst] = {cc, return_roots, trampoline_decl, nullptr, specsig}; - if (trim_may_error(ctx.params->trim)) - push_frames(ctx, ctx.linfo, mi); } } } @@ -5551,17 +5354,6 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, ArrayR } } if (!handled) { - if (trim_may_error(ctx.params->trim)) { - if (lival.constant) { - push_frames(ctx, ctx.linfo, (jl_method_instance_t*)lival.constant); - } - else { - errs() << "Dynamic call to unknown function"; - errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; - - print_stacktrace(ctx, ctx.params->trim); - } - } Value *r = emit_jlcall(ctx, jlinvoke_func, boxed(ctx, lival), argv, nargs, julia_call2); result = mark_julia_type(ctx, r, true, rt); } @@ -5621,12 +5413,6 @@ static jl_cgval_t emit_invoke_modify(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_ return mark_julia_type(ctx, oldnew, true, rt); } } - if (trim_may_error(ctx.params->trim)) { - errs() << "ERROR: dynamic invoke modify call to"; - jl_(args[0]); - errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; - print_stacktrace(ctx, ctx.params->trim); - } // emit function and arguments Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, nargs, julia_call); return mark_julia_type(ctx, callval, true, rt); @@ -5701,35 +5487,6 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo // special case for some known builtin not handled by emit_builtin_call auto it = builtin_func_map().find(builtin_fptr); if (it != builtin_func_map().end()) { - if (trim_may_error(ctx.params->trim)) { - bool may_dispatch = may_dispatch_builtins().count(builtin_fptr); - if (may_dispatch && f.constant == jl_builtin__apply_iterate && nargs >= 4) { - if (jl_subtype(argv[2].typ, (jl_value_t*)jl_builtin_type)) { - static jl_value_t *jl_dispatchfree_apply_iterate_type = NULL; - if (!jl_dispatchfree_apply_iterate_type) { - jl_value_t *types[5] = { - (jl_value_t *)jl_simplevector_type, - (jl_value_t *)jl_genericmemory_type, - (jl_value_t *)jl_array_type, - (jl_value_t *)jl_tuple_type, - (jl_value_t *)jl_namedtuple_type, - }; - jl_dispatchfree_apply_iterate_type = jl_as_global_root(jl_type_union(types, 5), 1); - } - for (size_t i = 3; i < nargs; i++) { - auto ai = argv[i].typ; - if (!jl_subtype(ai, jl_dispatchfree_apply_iterate_type)) - break; - } - may_dispatch = false; - } - } - if (may_dispatch) { - errs() << "ERROR: Dynamic call to builtin " << jl_symbol_name(((jl_datatype_t*)jl_typeof(f.constant))->name->name); - errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; - print_stacktrace(ctx, ctx.params->trim); - } - } Value *ret = emit_jlcall(ctx, it->second, Constant::getNullValue(ctx.types().T_prjlvalue), ArrayRef(argv).drop_front(), nargs - 1, julia_call); setName(ctx.emission_context, ret, it->second->name + "_ret"); return mark_julia_type(ctx, ret, true, rt); @@ -5745,11 +5502,6 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo fptr = ctx.builder.CreateCall(prepare_call(jlgetbuiltinfptr_func), {emit_typeof(ctx, f)}); cc = julia_call; } - if (trim_may_error(ctx.params->trim)) { - errs() << "ERROR: Dynamic call to unknown builtin"; - errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; - print_stacktrace(ctx, ctx.params->trim); - } Value *ret = emit_jlcall(ctx, fptr, nullptr, argv, nargs, cc); setName(ctx.emission_context, ret, "Builtin_ret"); return mark_julia_type(ctx, ret, true, rt); @@ -5770,40 +5522,6 @@ static jl_cgval_t emit_call(jl_codectx_t &ctx, jl_expr_t *ex, jl_value_t *rt, bo // TODO: else emit_oc_call } } - int failed_dispatch = !argv[0].constant; - if (ctx.params->trim != JL_TRIM_NO) { - // TODO: Implement the last-minute call resolution that used to be here - // in inference instead. - failed_dispatch = 1; - } - - if (failed_dispatch && trim_may_error(ctx.params->trim)) { - errs() << "Dynamic call to "; - jl_jmp_buf *old_buf = jl_get_safe_restore(); - jl_jmp_buf buf; - jl_set_safe_restore(&buf); - if (!jl_setjmp(buf, 0)) { - jl_static_show((JL_STREAM*)STDERR_FILENO, (jl_value_t*)args[0]); - jl_printf((JL_STREAM*)STDERR_FILENO,"("); - for (size_t i = 1; i < nargs; ++i) { - jl_value_t *typ = argv[i].typ; - if (!jl_is_concrete_type(typ)) // Print type in red - jl_printf((JL_STREAM*)STDERR_FILENO, "\x1b[31m"); - jl_static_show((JL_STREAM*)STDERR_FILENO, (jl_value_t*)argv[i].typ); - if (!jl_is_concrete_type(typ)) - jl_printf((JL_STREAM*)STDERR_FILENO, "\x1b[0m"); - if (i != nargs-1) - jl_printf((JL_STREAM*)STDERR_FILENO,", "); - } - jl_printf((JL_STREAM*)STDERR_FILENO,")\n"); - } - else { - jl_printf((JL_STREAM*)STDERR_FILENO, "\n!!! ERROR while printing error -- ABORTING !!!\n"); - } - jl_set_safe_restore(old_buf); - errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; - print_stacktrace(ctx, ctx.params->trim); - } // emit function and arguments Value *callval = emit_jlcall(ctx, jlapplygeneric_func, nullptr, argv, n_generic_args, julia_call); return mark_julia_type(ctx, callval, true, rt); @@ -6854,12 +6572,6 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr, ssize_t ssaidx_ ((jl_method_t*)source.constant)->nargs > 0 && jl_is_valid_oc_argtype((jl_tupletype_t*)argt.constant, (jl_method_t*)source.constant); - if (!can_optimize && trim_may_error(ctx.params->trim)) { - // if we know the return type, we can assume the result is of that type - errs() << "ERROR: Dynamic call to OpaqueClosure method\n"; - errs() << "In " << ctx.builder.getCurrentDebugLocation()->getFilename() << ":" << ctx.builder.getCurrentDebugLocation()->getLine() << "\n"; - print_stacktrace(ctx, ctx.params->trim); - } if (can_optimize) { jl_value_t *closure_t = NULL; @@ -7106,9 +6818,6 @@ Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Value *theFunc, Module * GlobalVariable::InternalLinkage, name, M); jl_init_function(f, params.TargetTriple); - if (trim_may_error(params.params->trim)) { - push_frames(ctx, ctx.linfo, jl_get_ci_mi(codeinst)); - } jl_name_jlfunc_args(params, f); //f->setAlwaysInline(); ctx.f = f; // for jl_Module diff --git a/src/init.c b/src/init.c index aada2c75ed7a6..333c469226fcf 100644 --- a/src/init.c +++ b/src/init.c @@ -739,8 +739,7 @@ JL_DLLEXPORT jl_cgparams_t jl_default_cgparams = { /* debug_info_level */ 0, // later jl_options.debug_level, /* safepoint_on_entry */ 1, /* gcstack_arg */ 1, - /* use_jlplt*/ 1, - /* trim */ 0 }; + /* use_jlplt*/ 1 }; static void init_global_mutexes(void) { JL_MUTEX_INIT(&jl_modules_mutex, "jl_modules_mutex"); diff --git a/src/jitlayers.h b/src/jitlayers.h index 139137d0ca477..c1699a5c1d913 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -244,7 +244,6 @@ struct jl_codegen_params_t { std::map ditypes; std::map llvmtypes; DenseMap mergedConstants; - llvm::MapVector> enqueuers; // Map from symbol name (in a certain library) to its GV in sysimg and the // DL handle address in the current session. StringMap> libMapGV; diff --git a/src/julia.h b/src/julia.h index 3206b4f1f033b..11049da643372 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2683,7 +2683,6 @@ typedef struct { int gcstack_arg; // Pass the ptls value as an argument with swiftself int use_jlplt; // Whether to use the Julia PLT mechanism or emit symbols directly - int trim; // can we emit dynamic dispatches? } jl_cgparams_t; extern JL_DLLEXPORT int jl_default_debug_info_kind; extern JL_DLLEXPORT jl_cgparams_t jl_default_cgparams; diff --git a/sysimage.mk b/sysimage.mk index ae6ce8699f417..55a28695acceb 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -57,10 +57,12 @@ COMPILER_SRCS := $(addprefix $(JULIAHOME)/, \ base/traits.jl \ base/refvalue.jl \ base/tuple.jl) -COMPILER_SRCS += $(shell find $(JULIAHOME)/Compiler/src -name \*.jl) +COMPILER_SRCS += $(shell find $(JULIAHOME)/Compiler/src -name \*.jl -and -not -name verifytrim.jl -and -not -name show.jl) # sort these to remove duplicates BASE_SRCS := $(sort $(shell find $(JULIAHOME)/base -name \*.jl -and -not -name sysimg.jl) \ - $(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl)) + $(shell find $(BUILDROOT)/base -name \*.jl -and -not -name sysimg.jl)) \ + $(JULIAHOME)/Compiler/src/ssair/show.jl \ + $(JULIAHOME)/Compiler/src/verifytrim.jl STDLIB_SRCS := $(JULIAHOME)/base/sysimg.jl $(SYSIMG_STDLIBS_SRCS) RELBUILDROOT := $(call rel_path,$(JULIAHOME)/base,$(BUILDROOT)/base)/ # <-- make sure this always has a trailing slash RELDATADIR := $(call rel_path,$(JULIAHOME)/base,$(build_datarootdir))/ # <-- make sure this always has a trailing slash From 10d75d25b095d5a70e824fb27987796e159ce071 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 27 Feb 2025 11:18:40 -0500 Subject: [PATCH 06/74] [Compiler] fix some cycle_fix_limited usage (#57545) Recompute some O(n) things a bit less on every statement. Fix an assert that ensured LimitedAccuracy does not accidentally get preserved when it should have been deleted: the (local) cache should not contain things that are marked as dead (RIP), as that was leading to much code not getting cached when it logically should have. Simplify the computation of frame_parent when the cycle_parent isn't needed. (cherry picked from commit 0366a2a5716193c00a6f447758551391e1f3da5a) --- Compiler/src/abstractinterpretation.jl | 16 +++----- Compiler/src/inferenceresult.jl | 3 +- Compiler/src/inferencestate.jl | 9 ++-- Compiler/src/ssair/irinterp.jl | 3 +- Compiler/src/typeinfer.jl | 57 +++++++++++++------------- Compiler/src/types.jl | 3 +- 6 files changed, 45 insertions(+), 46 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 73949068ce8c6..c280b4ee2ad08 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -286,19 +286,12 @@ function abstract_call_gf_by_type(interp::AbstractInterpreter, @nospecialize(fun state.rettype = Any end # if from_interprocedural added any pclimitations to the set inherited from the arguments, - # some of those may be part of our cycles, so those can be deleted now - # TODO: and those might need to be deleted later too if the cycle grows to include them? if isa(sv, InferenceState) # TODO (#48913) implement a proper recursion handling for irinterp: - # This works just because currently the `:terminate` condition guarantees that - # irinterp doesn't fail into unresolved cycles, but it's not a good solution. + # This works most of the time just because currently the `:terminate` condition often guarantees that + # irinterp doesn't fail into unresolved cycles, but it is not a good (or working) solution. # We should revisit this once we have a better story for handling cycles in irinterp. - if !isempty(sv.pclimitations) # remove self, if present - delete!(sv.pclimitations, sv) - for caller in callers_in_cycle(sv) - delete!(sv.pclimitations, caller) - end - end + delete!(sv.pclimitations, sv) # remove self, if present end else # there is unanalyzed candidate, widen type and effects to the top @@ -775,7 +768,7 @@ function edge_matches_sv(interp::AbstractInterpreter, frame::AbsIntState, # check in the cycle list first # all items in here are considered mutual parents of all others if !any(p::AbsIntState->matches_sv(p, sv), callers_in_cycle(frame)) - let parent = frame_parent(frame) + let parent = cycle_parent(frame) parent === nothing && return false (is_cached(parent) || frame_parent(parent) !== nothing) || return false matches_sv(parent, sv) || return false @@ -1379,6 +1372,7 @@ function const_prop_call(interp::AbstractInterpreter, inf_result.result = concrete_eval_result.rt inf_result.ipo_effects = concrete_eval_result.effects end + typ = inf_result.result return const_prop_result(inf_result) end diff --git a/Compiler/src/inferenceresult.jl b/Compiler/src/inferenceresult.jl index 7da96c4cc2e93..77f897e4035a5 100644 --- a/Compiler/src/inferenceresult.jl +++ b/Compiler/src/inferenceresult.jl @@ -183,7 +183,8 @@ function cache_lookup(𝕃::AbstractLattice, mi::MethodInstance, given_argtypes: method = mi.def::Method nargtypes = length(given_argtypes) for cached_result in cache - cached_result.linfo === mi || @goto next_cache + cached_result.tombstone && continue # ignore deleted entries (due to LimitedAccuracy) + cached_result.linfo === mi || continue cache_argtypes = cached_result.argtypes @assert length(cache_argtypes) == nargtypes "invalid `cache_argtypes` for `mi`" cache_overridden_by_const = cached_result.overridden_by_const::BitVector diff --git a/Compiler/src/inferencestate.jl b/Compiler/src/inferencestate.jl index 0ea0fc684b689..dbad9f76852e4 100644 --- a/Compiler/src/inferencestate.jl +++ b/Compiler/src/inferencestate.jl @@ -292,7 +292,7 @@ mutable struct InferenceState # IPO tracking of in-process work, shared with all frames given AbstractInterpreter callstack #::Vector{AbsIntState} - parentid::Int # index into callstack of the parent frame that originally added this frame (call frame_parent to extract the current parent of the SCC) + parentid::Int # index into callstack of the parent frame that originally added this frame (call cycle_parent to extract the current parent of the SCC) frameid::Int # index into callstack at which this object is found (or zero, if this is not a cached frame and has no parent) cycleid::Int # index into the callstack of the topmost frame in the cycle (all frames in the same cycle share the same cycleid) @@ -908,14 +908,17 @@ function frame_module(sv::AbsIntState) return def.module end -function frame_parent(sv::InferenceState) +frame_parent(sv::AbsIntState) = sv.parentid == 0 ? nothing : (sv.callstack::Vector{AbsIntState})[sv.parentid] + +function cycle_parent(sv::InferenceState) sv.parentid == 0 && return nothing callstack = sv.callstack::Vector{AbsIntState} sv = callstack[sv.cycleid]::InferenceState sv.parentid == 0 && return nothing return callstack[sv.parentid] end -frame_parent(sv::IRInterpretationState) = sv.parentid == 0 ? nothing : (sv.callstack::Vector{AbsIntState})[sv.parentid] +cycle_parent(sv::IRInterpretationState) = frame_parent(sv) + # add the orphan child to the parent and the parent to the child function assign_parentchild!(child::InferenceState, parent::AbsIntState) diff --git a/Compiler/src/ssair/irinterp.jl b/Compiler/src/ssair/irinterp.jl index a4969e81828cc..084f28f0aa523 100644 --- a/Compiler/src/ssair/irinterp.jl +++ b/Compiler/src/ssair/irinterp.jl @@ -1,7 +1,7 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license function collect_limitations!(@nospecialize(typ), ::IRInterpretationState) - @assert !isa(typ, LimitedAccuracy) "irinterp is unable to handle heavy recursion" + @assert !isa(typ, LimitedAccuracy) "irinterp is unable to handle heavy recursion correctly" return typ end @@ -212,6 +212,7 @@ function reprocess_instruction!(interp::AbstractInterpreter, inst::Instruction, else rt = argextype(stmt, irsv.ir) end + @assert !(rt isa LimitedAccuracy) if rt !== nothing if has_flag(inst, IR_FLAG_UNUSED) # Don't bother checking the type if we know it's unused diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 2ae3242be4027..7b0cf09601c4a 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -193,7 +193,7 @@ function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstan end function finish_nocycle(::AbstractInterpreter, frame::InferenceState) - finishinfer!(frame, frame.interp) + finishinfer!(frame, frame.interp, frame.cycleid) opt = frame.result.src if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` optimize(frame.interp, opt, frame.result) @@ -230,7 +230,7 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei for frameid = cycleid:length(frames) caller = frames[frameid]::InferenceState adjust_cycle_frame!(caller, cycle_valid_worlds, cycle_valid_effects) - finishinfer!(caller, caller.interp) + finishinfer!(caller, caller.interp, cycleid) end for frameid = cycleid:length(frames) caller = frames[frameid]::InferenceState @@ -310,26 +310,21 @@ function cache_result!(interp::AbstractInterpreter, result::InferenceResult, ci: return true end -function cycle_fix_limited(@nospecialize(typ), sv::InferenceState) +function cycle_fix_limited(@nospecialize(typ), sv::InferenceState, cycleid::Int) if typ isa LimitedAccuracy - if sv.parentid === 0 - # we might have introduced a limit marker, but we should know it must be sv and other callers_in_cycle - #@assert !isempty(callers_in_cycle(sv)) - # FIXME: this assert fails, appearing to indicate there is a bug in filtering this list earlier. - # In particular (during doctests for example), during inference of - # show(Base.IOContext{Base.GenericIOBuffer{Memory{UInt8}}}, Base.Multimedia.MIME{:var"text/plain"}, LinearAlgebra.BunchKaufman{Float64, Array{Float64, 2}, Array{Int64, 1}}) - # we observed one of the ssavaluetypes here to be Core.Compiler.LimitedAccuracy(typ=Any, causes=Core.Compiler.IdSet(getproperty(LinearAlgebra.BunchKaufman{Float64, Array{Float64, 2}, Array{Int64, 1}}, Symbol))) - return typ.typ - end - causes = copy(typ.causes) - delete!(causes, sv) - for caller in callers_in_cycle(sv) - delete!(causes, caller) - end - if isempty(causes) - return typ.typ + frames = sv.callstack::Vector{AbsIntState} + causes = typ.causes + for frameid = cycleid:length(frames) + caller = frames[frameid]::InferenceState + caller in causes || continue + causes === typ.causes && (causes = copy(causes)) + pop!(causes, caller) + if isempty(causes) + return typ.typ + end end - if length(causes) != length(typ.causes) + @assert sv.parentid != 0 + if causes !== typ.causes return LimitedAccuracy(typ.typ, causes) end end @@ -437,20 +432,23 @@ const empty_edges = Core.svec() # inference completed on `me` # update the MethodInstance -function finishinfer!(me::InferenceState, interp::AbstractInterpreter) +function finishinfer!(me::InferenceState, interp::AbstractInterpreter, cycleid::Int) # prepare to run optimization passes on fulltree @assert isempty(me.ip) # inspect whether our inference had a limited result accuracy, # else it may be suitable to cache - bestguess = me.bestguess = cycle_fix_limited(me.bestguess, me) - exc_bestguess = me.exc_bestguess = cycle_fix_limited(me.exc_bestguess, me) + bestguess = me.bestguess = cycle_fix_limited(me.bestguess, me, cycleid) + exc_bestguess = me.exc_bestguess = cycle_fix_limited(me.exc_bestguess, me, cycleid) limited_ret = bestguess isa LimitedAccuracy || exc_bestguess isa LimitedAccuracy limited_src = false - if !limited_ret + if limited_ret + @assert me.parentid != 0 + else gt = me.ssavaluetypes for j = 1:length(gt) - gt[j] = gtj = cycle_fix_limited(gt[j], me) - if gtj isa LimitedAccuracy && me.parentid != 0 + gt[j] = gtj = cycle_fix_limited(gt[j], me, cycleid) + if gtj isa LimitedAccuracy + @assert me.parentid != 0 limited_src = true break end @@ -472,6 +470,7 @@ function finishinfer!(me::InferenceState, interp::AbstractInterpreter) # a parent may be cached still, but not this intermediate work: # we can throw everything else away now result.src = nothing + result.tombstone = true me.cache_mode = CACHE_MODE_NULL set_inlineable!(me.src, false) elseif limited_src @@ -712,7 +711,7 @@ function merge_call_chain!(::AbstractInterpreter, parent::InferenceState, child: add_cycle_backedge!(parent, child) parent.cycleid === ancestorid && break child = parent - parent = frame_parent(child)::InferenceState + parent = cycle_parent(child)::InferenceState end # ensure that walking the callstack has the same cycleid (DAG) for frameid = reverse(ancestorid:length(frames)) @@ -748,7 +747,7 @@ end # returned instead. function resolve_call_cycle!(interp::AbstractInterpreter, mi::MethodInstance, parent::AbsIntState) # TODO (#48913) implement a proper recursion handling for irinterp: - # This works currently just because the irinterp code doesn't get used much with + # This works most of the time currently just because the irinterp code doesn't get used much with # `@assume_effects`, so it never sees a cycle normally, but that may not be a sustainable solution. parent isa InferenceState || return false frames = parent.callstack::Vector{AbsIntState} @@ -760,7 +759,7 @@ function resolve_call_cycle!(interp::AbstractInterpreter, mi::MethodInstance, pa if is_same_frame(interp, mi, frame) if uncached # our attempt to speculate into a constant call lead to an undesired self-cycle - # that cannot be converged: poison our call-stack (up to the discovered duplicate frame) + # that cannot be converged: if necessary, poison our call-stack (up to the discovered duplicate frame) # with the limited flag and abort (set return type to Any) now poison_callstack!(parent, frame) return true diff --git a/Compiler/src/types.jl b/Compiler/src/types.jl index 6ffb5402682f3..306728a8a21e9 100644 --- a/Compiler/src/types.jl +++ b/Compiler/src/types.jl @@ -106,6 +106,7 @@ mutable struct InferenceResult effects::Effects # if optimization is finished analysis_results::AnalysisResults # AnalysisResults with e.g. result::ArgEscapeCache if optimized, otherwise NULL_ANALYSIS_RESULTS is_src_volatile::Bool # `src` has been cached globally as the compressed format already, allowing `src` to be used destructively + tombstone::Bool #=== uninitialized fields ===# ci::CodeInstance # CodeInstance if this result may be added to the cache @@ -116,7 +117,7 @@ mutable struct InferenceResult ipo_effects = effects = Effects() analysis_results = NULL_ANALYSIS_RESULTS return new(mi, argtypes, overridden_by_const, result, exc_result, src, - valid_worlds, ipo_effects, effects, analysis_results, #=is_src_volatile=#false) + valid_worlds, ipo_effects, effects, analysis_results, #=is_src_volatile=#false, false) end end function InferenceResult(mi::MethodInstance, 𝕃::AbstractLattice=fallback_lattice) From 828c7ccb1b779056e3e87e6eb4cb5024969fc4ea Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Thu, 27 Feb 2025 11:20:11 -0500 Subject: [PATCH 07/74] some updates for trimming (#57531) - disable load path setup - remove upstreamed change to utf8proc_map - fix warning for accessing mod.main in the wrong world - remove binding backedges when trimming (cherry picked from commit d77c24f009b93577bcf254908e639c9f149f2e4e) --- contrib/juliac-buildscript.jl | 15 ++++++--------- src/staticdata.c | 1 + 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index c23b679272b1e..7d25e85ff4c5c 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -33,6 +33,9 @@ end JuliaSyntax.enable_in_core!() = nothing init_active_project() = ACTIVE_PROJECT[] = nothing set_active_project(projfile::Union{AbstractString,Nothing}) = ACTIVE_PROJECT[] = projfile + init_depot_path() = nothing + init_load_path() = nothing + init_active_project() = nothing disable_library_threading() = nothing start_profile_listener() = nothing @inline function invokelatest(f::F, args...; kwargs...) where F @@ -132,15 +135,8 @@ end mapreduce_empty(::typeof(abs), op::F, T) where {F} = abs(reduce_empty(op, T)) mapreduce_empty(::typeof(abs2), op::F, T) where {F} = abs2(reduce_empty(op, T)) end -@eval Base.Unicode begin - function utf8proc_map(str::Union{String,SubString{String}}, options::Integer, chartransform::F = identity) where F - nwords = utf8proc_decompose(str, options, C_NULL, 0, chartransform) - buffer = Base.StringVector(nwords*4) - nwords = utf8proc_decompose(str, options, buffer, nwords, chartransform) - nbytes = ccall(:utf8proc_reencode, Int, (Ptr{UInt8}, Int, Cint), buffer, nwords, options) - nbytes < 0 && utf8proc_error(nbytes) - return String(resize!(buffer, nbytes)) - end +@eval Base.Sys begin + __init_build() = nothing end @eval Base.GMP begin function __init__() @@ -202,6 +198,7 @@ let mod = Base.include(Base.__toplevel__, inputfile) if !isa(mod, Module) mod = Main end + Core.@latestworld if output_type == "--output-exe" && isdefined(mod, :main) && !add_ccallables entrypoint(mod.main, ()) end diff --git a/src/staticdata.c b/src/staticdata.c index 2be949d408756..666bf2d8ccd29 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -824,6 +824,7 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ // ... or point to Base functions accessed by the runtime (m == jl_base_module && (!strcmp(jl_symbol_name(b->globalref->name), "wait") || !strcmp(jl_symbol_name(b->globalref->name), "task_done_hook"))))) { + record_field_change((jl_value_t**)&b->backedges, NULL); jl_queue_for_serialization(s, b); } } From c5d0b6728f62d3c2c23b184533d81e0fd4283a8b Mon Sep 17 00:00:00 2001 From: Timothy Date: Fri, 28 Feb 2025 00:50:37 +0800 Subject: [PATCH 08/74] Bump JuliaSyntaxHighlighting (#57526) This updates the pin to point to main rather than a PR again, and also includes the commit: ``` b7a1c63 * Guard against assumptions that children exist ``` Fixes #57402 (cherry picked from commit 61de3a446ecb9c07ba206770a8999859390370a5) --- .../md5 | 1 - .../sha512 | 1 - .../md5 | 1 + .../sha512 | 1 + stdlib/JuliaSyntaxHighlighting.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-2680c8bde1aa274f25d7a434c645f16b3a1ee731.tar.gz/md5 delete mode 100644 deps/checksums/JuliaSyntaxHighlighting-2680c8bde1aa274f25d7a434c645f16b3a1ee731.tar.gz/sha512 create mode 100644 deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 create mode 100644 deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 diff --git a/deps/checksums/JuliaSyntaxHighlighting-2680c8bde1aa274f25d7a434c645f16b3a1ee731.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-2680c8bde1aa274f25d7a434c645f16b3a1ee731.tar.gz/md5 deleted file mode 100644 index 30284ccf352d4..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-2680c8bde1aa274f25d7a434c645f16b3a1ee731.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -187f155c32a79f57a89e31e672d2d8c5 diff --git a/deps/checksums/JuliaSyntaxHighlighting-2680c8bde1aa274f25d7a434c645f16b3a1ee731.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-2680c8bde1aa274f25d7a434c645f16b3a1ee731.tar.gz/sha512 deleted file mode 100644 index bdce410b84d69..0000000000000 --- a/deps/checksums/JuliaSyntaxHighlighting-2680c8bde1aa274f25d7a434c645f16b3a1ee731.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -69347af996d77b88b5e5b6e44ff046e9197775a66802a0da6fb5fcbf9e5ca533566955c8435bc25490f6ca0c002b4c1effcddaf932b7eb91e00a8f99554b7b8d diff --git a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 b/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 new file mode 100644 index 0000000000000..2a4b55e15ab2d --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/md5 @@ -0,0 +1 @@ +ed0ccc4434fc70b06e8ea1ddb8141511 diff --git a/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 b/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 new file mode 100644 index 0000000000000..456f1ee64ca0b --- /dev/null +++ b/deps/checksums/JuliaSyntaxHighlighting-b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f.tar.gz/sha512 @@ -0,0 +1 @@ +04efea853a1c1bfbf5baf4d2908ce492a5ff3029bca73a004280aa116157b6b678a5f9fd6a115f9c57a625d0841d3fb96c8d68ec467e5bc4a743272bee84c8c7 diff --git a/stdlib/JuliaSyntaxHighlighting.version b/stdlib/JuliaSyntaxHighlighting.version index 14eb1cedf49a4..1c9bfb131dc0f 100644 --- a/stdlib/JuliaSyntaxHighlighting.version +++ b/stdlib/JuliaSyntaxHighlighting.version @@ -1,4 +1,4 @@ JULIASYNTAXHIGHLIGHTING_BRANCH = main -JULIASYNTAXHIGHLIGHTING_SHA1 = 2680c8bde1aa274f25d7a434c645f16b3a1ee731 +JULIASYNTAXHIGHLIGHTING_SHA1 = b7a1c636d3e9690bfbbfe917bb20f6cb112a3e6f JULIASYNTAXHIGHLIGHTING_GIT_URL := https://github.com/julialang/JuliaSyntaxHighlighting.jl.git JULIASYNTAXHIGHLIGHTING_TAR_URL = https://api.github.com/repos/julialang/JuliaSyntaxHighlighting.jl/tarball/$1 From c450cffb9ef80a936e81dcccd395b39f761cd9d5 Mon Sep 17 00:00:00 2001 From: Denis Barucic Date: Thu, 27 Feb 2025 20:49:06 +0100 Subject: [PATCH 09/74] Docs: `circshift!(::AbstractVector, ::Integer)` (#57539) Closes #46016 (cherry picked from commit b0323ab00fe756c5a30aeed80186e6e547417421) --- base/abstractarray.jl | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/base/abstractarray.jl b/base/abstractarray.jl index 1ab78a55c93b5..62bf0901a0507 100644 --- a/base/abstractarray.jl +++ b/base/abstractarray.jl @@ -3667,7 +3667,31 @@ function _keepat!(a::AbstractVector, m::AbstractVector{Bool}) deleteat!(a, j:lastindex(a)) end -## 1-d circshift ## +""" + circshift!(a::AbstractVector, shift::Integer) + +Circularly shift, or rotate, the data in vector `a` by `shift` positions. + +# Examples + +```jldoctest +julia> circshift!([1, 2, 3, 4, 5], 2) +5-element Vector{Int64}: + 4 + 5 + 1 + 2 + 3 + +julia> circshift!([1, 2, 3, 4, 5], -2) +5-element Vector{Int64}: + 3 + 4 + 5 + 1 + 2 +``` +""" function circshift!(a::AbstractVector, shift::Integer) n = length(a) n == 0 && return a From e2884bc19071dda3eba400a47fc71b89b4484fff Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Thu, 27 Feb 2025 15:52:38 -0800 Subject: [PATCH 10/74] lowering: Allow chaining of `>:` in `where` (#57554) This change removes an inconsistency between `>:` and `<:`. We were parsing `where {A>:B>:C}` forms, but not recognizing them in lowering. Before: ``` julia> Vector{T} where Int<:T<:Number Vector{T} where Int64<:T<:Number julia> Vector{T} where Number>:T>:Int ERROR: syntax: invalid bounds in "where" around REPL[14]:1 ``` After: ``` julia> Vector{T} where Number>:T>:Int Vector{T} where Int64<:T<:Number ``` (cherry picked from commit 2e57730aa253b4d8dc90be56a4282bbece045ccc) --- src/julia-syntax.scm | 12 ++++++++---- test/syntax.jl | 4 ++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 47cbc9dec1503..51753e91b928d 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -194,10 +194,14 @@ (cond ((atom? e) (list (check-sym e) #f #f)) ((eq? (car e) 'var-bounds) (cdr e)) ((and (eq? (car e) 'comparison) (length= e 6)) - (cons (check-sym (cadddr e)) - (cond ((and (eq? (caddr e) '|<:|) (eq? (caddr (cddr e)) '|<:|)) - (list (cadr e) (last e))) - (else (error "invalid bounds in \"where\""))))) + (let* ((lhs (list-ref e 1)) + (rel (list-ref e 2)) + (t (check-sym (list-ref e 3))) + (rel-same (eq? rel (list-ref e 4))) + (rhs (list-ref e 5))) + (cond ((and rel-same (eq? rel '|<:|)) (list t lhs rhs)) + ((and rel-same (eq? rel '|>:|)) (list t rhs lhs)) + (else (error "invalid bounds in \"where\""))))) ((eq? (car e) '|<:|) (list (check-sym (cadr e)) #f (caddr e))) ((eq? (car e) '|>:|) diff --git a/test/syntax.jl b/test/syntax.jl index 13d0d82e20d06..5ea705cddcdc7 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -1292,6 +1292,10 @@ let args = (Int, Any) @test >:(reverse(args)...) end +# Chaining of <: and >: in `where` +@test isa(Vector{T} where Int<:T<:Number, UnionAll) +@test isa(Vector{T} where Number>:T>:Int, UnionAll) + # issue #25947 let getindex = 0, setindex! = 1, colon = 2, vcat = 3, hcat = 4, hvcat = 5 a = [10,9,8] From e878f5cede24c281f77d0f003485a6785054fc09 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 27 Feb 2025 21:21:25 -0500 Subject: [PATCH 11/74] bpart: Check whether the replaced binding is a type before assuming (#57558) Somebody may replace the type with an int. Fixes #57515. (cherry picked from commit bfe9c2febe859530b2c9bec506ca3bba41cd5c61) --- base/show.jl | 11 +++++++---- test/rebinding.jl | 7 +++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/base/show.jl b/base/show.jl index 0cc16b11cb298..8c9f18c2e01fe 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1051,10 +1051,13 @@ function check_world_bounded(tn::Core.TypeName) isdefined(bnd, :partitions) || return nothing partition = @atomic bnd.partitions while true - if is_defined_const_binding(binding_kind(partition)) && partition_restriction(partition) <: tn.wrapper - max_world = @atomic partition.max_world - max_world == typemax(UInt) && return nothing - return Int(partition.min_world):Int(max_world) + if is_defined_const_binding(binding_kind(partition)) + cval = partition_restriction(partition) + if isa(cval, Type) && cval <: tn.wrapper + max_world = @atomic partition.max_world + max_world == typemax(UInt) && return nothing + return Int(partition.min_world):Int(max_world) + end end isdefined(partition, :next) || return nothing partition = @atomic partition.next diff --git a/test/rebinding.jl b/test/rebinding.jl index c6feabdd13b59..e9dfa38261de5 100644 --- a/test/rebinding.jl +++ b/test/rebinding.jl @@ -18,6 +18,13 @@ module Rebinding @test Base.binding_kind(@__MODULE__, :Foo) == Base.BINDING_KIND_GUARD @test contains(repr(x), "@world") + # Test that it still works if Foo is redefined to a non-type + const Foo = 1 + + @test Base.binding_kind(@__MODULE__, :Foo) == Base.BINDING_KIND_CONST + @test contains(repr(x), "@world") + Base.delete_binding(@__MODULE__, :Foo) + struct Foo x::Int end From c9d2a273a66ca28259eaca5dc4ee4b2992ec99e5 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 27 Feb 2025 21:51:26 -0500 Subject: [PATCH 12/74] Walk to imports in `isconst(::GlobalRef)` (#57563) This restores 1.11 behavior. However, I find this function a bit problematic, since it is asking whether the binding is `constant` in a particular sense, but not whether it is `const` (in the sense of having been declared with the keyword or equivalent), so the shortening is confusing. However, we don't need to address that now. Fixes #57475. (cherry picked from commit 1ff98a534badb1ad40d96aad4b5aa37748de5f0c) --- src/module.c | 3 +-- test/reflection.jl | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/module.c b/src/module.c index 848ced7058af1..31fd9b05739fd 100644 --- a/src/module.c +++ b/src/module.c @@ -1370,8 +1370,7 @@ JL_DLLEXPORT int jl_globalref_is_const(jl_globalref_t *gr) if (!b) b = jl_get_module_binding(gr->mod, gr->name, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (!bpart) - return 0; + jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); return jl_bkind_is_some_constant(jl_binding_kind(bpart)); } diff --git a/test/reflection.jl b/test/reflection.jl index 57c32d19de629..a0e9d96f044be 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -110,6 +110,7 @@ not_const = 1 @test isconst(@__MODULE__, :a_const) == true @test isconst(Base, :pi) == true @test isconst(@__MODULE__, :pi) == true +@test isconst(GlobalRef(@__MODULE__, :pi)) == true @test isconst(@__MODULE__, :not_const) == false @test isconst(@__MODULE__, :is_not_defined) == false From 0a8b382e7bfa867e7eef402fe4ef1d3a2fc017b4 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Fri, 28 Feb 2025 02:00:41 -0500 Subject: [PATCH 13/74] Always disallow assignments to constants (#57567) We have historically allowed the following without warning or error: ``` const x = 1 x = 1 ``` As well as ``` const x = 1 x = 2 ``` with a UB warning. In 1.12, we made the second case error, as part of the general binding partition changes, but did not touch the first case, because it did not have the warning. I think this made a reasonable amount of sense, because we essentially treated constants as special kinds of mutable globals (to which assignment happened to be UB). However, in the 1.12+ design, constants and globals are quite sepearate beasts, and I think it makes sense to extend the error to the egal case also, even if it is technically more breaking. In fact, I already thought that's what I did when I implemented the new effect model for global assignment, causing #57566. I can't think of a legitimate reason to keep this special case. For those who want to do binding replacement, the `const` keyword is mandatory, so the assignment is now literally always a semantic no-op or an error. Fixes #57566. (cherry picked from commit 8f00a51f074e5442546d48ba5d8f2346ed7f2a8c) --- src/module.c | 19 +++++++------------ test/syntax.jl | 7 ++++++- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/module.c b/src/module.c index 31fd9b05739fd..ee28c24798f95 100644 --- a/src/module.c +++ b/src/module.c @@ -545,14 +545,19 @@ JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module { jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); enum jl_partition_kind kind = jl_binding_kind(bpart); - if (kind != BINDING_KIND_GLOBAL && kind != BINDING_KIND_DECLARED && !jl_bkind_is_some_constant(kind)) { + if (kind != BINDING_KIND_GLOBAL && kind != BINDING_KIND_DECLARED) { if (jl_bkind_is_some_guard(kind)) { jl_errorf("Global %s.%s does not exist and cannot be assigned.\n" "Note: Julia 1.9 and 1.10 inadvertently omitted this error check (#56933).\n" "Hint: Declare it using `global %s` inside `%s` before attempting assignment.", jl_symbol_name(m->name), jl_symbol_name(s), jl_symbol_name(s), jl_symbol_name(m->name)); - } else { + } + else if (jl_bkind_is_some_constant(kind)) { + jl_errorf("invalid assignment to constant %s.%s. This redefinition may be permitted using the `const` keyword.", + jl_symbol_name(m->name), jl_symbol_name(s)); + } + else { jl_module_t *from = jl_binding_dbgmodule(b, m, s); if (from == m) jl_errorf("cannot assign a value to imported variable %s.%s", @@ -1448,16 +1453,6 @@ jl_value_t *jl_check_binding_assign_value(jl_binding_t *b JL_PROPAGATES_ROOT, jl JL_GC_PUSH1(&rhs); // callee-rooted jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); enum jl_partition_kind kind = jl_binding_kind(bpart); - if (jl_bkind_is_some_constant(kind)) { - jl_value_t *old = bpart->restriction; - JL_GC_PROMISE_ROOTED(old); - if (jl_egal(rhs, old)) { - JL_GC_POP(); - return NULL; - } - jl_errorf("invalid assignment to constant %s.%s. This redefinition may be permitted using the `const` keyword.", - jl_symbol_name(mod->name), jl_symbol_name(var)); - } assert(kind == BINDING_KIND_DECLARED || kind == BINDING_KIND_GLOBAL); jl_value_t *old_ty = kind == BINDING_KIND_DECLARED ? (jl_value_t*)jl_any_type : bpart->restriction; JL_GC_PROMISE_ROOTED(old_ty); diff --git a/test/syntax.jl b/test/syntax.jl index 5ea705cddcdc7..8125f44cfe693 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -3969,8 +3969,13 @@ end # Test trying to define a constant and then trying to assign to the same value module AssignConstValueTest + using Test const x = 1 - x = 1 + @test_throws ErrorException @eval x = 1 + @test_throws ErrorException @eval begin + @Base.Experimental.force_compile + global x = 1 + end end @test isconst(AssignConstValueTest, :x) From 5323469d051a4531973b81fad3d7141aeaf5c259 Mon Sep 17 00:00:00 2001 From: Adam Wheeler Date: Fri, 28 Feb 2025 02:24:08 -0500 Subject: [PATCH 14/74] Group import hints (#56753) This is my attempt to resolve #53000. ``` julia> solve ERROR: UndefVarError: `solve` not defined in `Main` Suggestion: check for spelling errors or missing imports. Hint: a global variable of this name also exists in CommonSolve. - Also exported by SciMLBase. - Also exported by DiffEqBase. - Also exported by JumpProcesses. - Also exported by LinearSolve. - Also exported by BracketingNonlinearSolve (loaded but not imported in Main). - Also exported by SimpleNonlinearSolve. - Also exported by NonlinearSolve. - Also exported by OrdinaryDiffEqLowStorageRK (loaded but not imported in Main). - Also exported by OrdinaryDiffEqSSPRK (loaded but not imported in Main). - Also exported by OrdinaryDiffEqVerner (loaded but not imported in Main). - Also exported by OrdinaryDiffEqBDF (loaded but not imported in Main). - Also exported by OrdinaryDiffEqTsit5 (loaded but not imported in Main). - Also exported by OrdinaryDiffEqRosenbrock (loaded but not imported in Main). - Also exported by OrdinaryDiffEqDefault (loaded but not imported in Main). - Also exported by Sundials. ``` I would have beefed up the test case, but can't follow how it works. --------- Co-authored-by: Alex Arslan Co-authored-by: Ian Butterworth Co-authored-by: Jameson Nash (cherry picked from commit 5f13cd2ddc058eb46ccb1b580c82f506b1b7dfe5) --- stdlib/REPL/src/REPL.jl | 71 ++++++++++++++++++++++++++++------------ stdlib/REPL/test/repl.jl | 43 ++++++++++++++++++++---- 2 files changed, 86 insertions(+), 28 deletions(-) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 561e4bcd4feb2..bbc2c9a6d44b0 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -55,36 +55,65 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) else scope = undef end - if scope !== Base && !_UndefVarError_warnfor(io, Base, var) - warned = false - for m in Base.loaded_modules_order - m === Core && continue - m === Base && continue - m === Main && continue - m === scope && continue - warned |= _UndefVarError_warnfor(io, m, var) + if scope !== Base + warned = _UndefVarError_warnfor(io, [Base], var) + + if !warned + modules_to_check = (m for m in Base.loaded_modules_order + if m !== Core && m !== Base && m !== Main && m !== scope) + warned |= _UndefVarError_warnfor(io, modules_to_check, var) end - warned || - _UndefVarError_warnfor(io, Core, var) || - _UndefVarError_warnfor(io, Main, var) + + warned || _UndefVarError_warnfor(io, [Core, Main], var) end return nothing end -function _UndefVarError_warnfor(io::IO, m::Module, var::Symbol) - (Base.isexported(m, var) || Base.ispublic(m, var)) || return false +function _UndefVarError_warnfor(io::IO, modules, var::Symbol) active_mod = Base.active_module() - print(io, "\nHint: ") - if isdefined(active_mod, Symbol(m)) - print(io, "a global variable of this name also exists in $m.") - else - if Symbol(m) == var - print(io, "$m is loaded but not imported in the active module $active_mod.") + + warned = false + # collect modules which export or make public the variable by + # the module in which the variable is defined + to_warn_about = Dict{Module, Vector{Module}}() + for m in modules + # only include in info if binding has a value and is exported or public + if !Base.isdefined(m, var) || (!Base.isexported(m, var) && !Base.ispublic(m, var)) + continue + end + warned = true + + # handle case where the undefined variable is the name of a loaded module + if Symbol(m) == var && !isdefined(active_mod, var) + print(io, "\nHint: $m is loaded but not imported in the active module $active_mod.") + continue + end + + binding_m = Base.binding_module(m, var) + if !haskey(to_warn_about, binding_m) + to_warn_about[binding_m] = [m] else - print(io, "a global variable of this name may be made accessible by importing $m in the current active module $active_mod") + push!(to_warn_about[binding_m], m) + end + end + + for (binding_m, modules) in pairs(to_warn_about) + print(io, "\nHint: a global variable of this name also exists in ", binding_m, ".") + for m in modules + m == binding_m && continue + how_available = if Base.isexported(m, var) + "exported by" + elseif Base.ispublic(m, var) + "declared public in" + end + print(io, "\n - Also $how_available $m") + if !isdefined(active_mod, nameof(m)) || (getproperty(active_mod, nameof(m)) !== m) + print(io, " (loaded but not imported in $active_mod)") + end + print(io, ".") end end - return true + return warned end function __init__() diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index c1c5c7844bc96..69f0cb846d8d7 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1833,23 +1833,52 @@ fake_repl() do stdin_write, stdout_read, repl @test contains(txt, "Some type information was truncated. Use `show(err)` to see complete types.") end -try # test the functionality of `UndefVarError_hint` against `Base.remove_linenums!` +try # test the functionality of `UndefVarError_hint` @assert isempty(Base.Experimental._hint_handlers) Base.Experimental.register_error_hint(REPL.UndefVarError_hint, UndefVarError) - # check the requirement to trigger the hint via `UndefVarError_hint` - @test !isdefined(Main, :remove_linenums!) && Base.ispublic(Base, :remove_linenums!) - fake_repl() do stdin_write, stdout_read, repl backend = REPL.REPLBackend() repltask = @async REPL.run_repl(repl; backend) - write(stdin_write, - "remove_linenums!\n\"ZZZZZ\"\n") + write(stdin_write, """ + module A53000 + export f + f() = 0.0 + end + + module C_outer_53000 + import ..A53000: f + public f + + module C_inner_53000 + import ..C_outer_53000: f + export f + end + end + + module D_53000 + public f + f() = 1.0 + end + + C_inner_53000 = "I'm a decoy with the same name as C_inner_53000!" + + append!(Base.loaded_modules_order, [A53000, C_outer_53000, C_outer_53000.C_inner_53000, D_53000]) + f + """ + ) + write(stdin_write, "\nZZZZZ\n") txt = readuntil(stdout_read, "ZZZZZ") write(stdin_write, '\x04') wait(repltask) - @test occursin("Hint: a global variable of this name also exists in Base.", txt) + @test occursin("Hint: a global variable of this name also exists in Main.A53000.", txt) + @test occursin("Hint: a global variable of this name also exists in Main.D_53000.", txt) + @test occursin("- Also declared public in Main.C_outer_53000.", txt) + @test occursin("- Also exported by Main.C_outer_53000.C_inner_53000 (loaded but not imported in Main).", txt) end +catch e + # fail test if error + @test false finally empty!(Base.Experimental._hint_handlers) end From 87d9c236008cd7ce8fe5963e59fd68527befb287 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Fri, 28 Feb 2025 08:34:24 -0500 Subject: [PATCH 15/74] Remove `jl_init__threading` and `jl_init_with_image__threading` (#57561) These were supposed to be removed in Julia 1.8 apparently (#40730) (cherry picked from commit 2dd4cdf2256c685cedcf33df58c210dde43a5b36) --- src/jl_exported_funcs.inc | 2 -- src/jlapi.c | 13 ------------- 2 files changed, 15 deletions(-) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 664e1270c7381..27d2e949bb569 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -246,8 +246,6 @@ XX(jl_init_options) \ XX(jl_init_restored_module) \ XX(jl_init_with_image) \ - XX(jl_init_with_image__threading) \ - XX(jl_init__threading) \ XX(jl_install_sigint_handler) \ XX(jl_instantiate_type_in_env) \ XX(jl_instantiate_unionall) \ diff --git a/src/jlapi.c b/src/jlapi.c index e205a4a4dc723..53585555b8a23 100644 --- a/src/jlapi.c +++ b/src/jlapi.c @@ -119,19 +119,6 @@ JL_DLLEXPORT void jl_init(void) free(libbindir); } -// HACK: remove this for Julia 1.8 (see ) -JL_DLLEXPORT void jl_init__threading(void) -{ - jl_init(); -} - -// HACK: remove this for Julia 1.8 (see ) -JL_DLLEXPORT void jl_init_with_image__threading(const char *julia_bindir, - const char *image_relative_path) -{ - jl_init_with_image(julia_bindir, image_relative_path); -} - static void _jl_exception_clear(jl_task_t *ct) JL_NOTSAFEPOINT { ct->ptls->previous_exception = NULL; From d27180556d49719b5ac8b5819d6f61743f21cf00 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 1 Mar 2025 00:09:38 -0500 Subject: [PATCH 16/74] Make no-body `function` declaration implicitly `global` (#57562) These were the intended semantics of #57311 (and matches what it used to do in 1.11). Note however that this differs from the body-ful form, which now always tries to extend. Fixes #57546. Note that this implementation is slightly inefficient since it goes through a binding replacement. However, there's a change coming down the line which optimizes these replacements. I do think it should eventually be refactored to just create the binding directly, but that's a little bit of a larger change. (cherry picked from commit 7fa0c13b934a86e205c95742fd974f6158651830) --- src/julia-syntax.scm | 10 ++++++++-- test/syntax.jl | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 51753e91b928d..199119d4b1ee8 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1185,7 +1185,9 @@ (cond ((and (length= e 2) (or (symbol? name) (globalref? name))) (if (not (valid-name? name)) (error (string "invalid function name \"" name "\""))) - `(method ,name)) + (if (globalref? name) + `(block (global ,name) (method ,name)) + `(block (global-if-global ,name) (method ,name)))) ((not (pair? name)) e) ((eq? (car name) 'call) (let* ((raw-typevars (or where '())) @@ -3129,6 +3131,10 @@ (if (eq? (var-kind (cadr e) scope) 'local) (if (length= e 2) (null) `(= ,@(cdr e))) `(const ,@(cdr e)))) + ((eq? (car e) 'global-if-global) + (if (eq? (var-kind (cadr e) scope) 'local) + '(null) + `(global ,@(cdr e)))) ((memq (car e) '(local local-def)) (check-valid-name (cadr e)) ;; remove local decls @@ -3761,7 +3767,7 @@ f(x) = yt(x) (Set '(quote top core lineinfo line inert local-def unnecessary copyast meta inbounds boundscheck loopinfo decl aliasscope popaliasscope thunk with-static-parameters toplevel-only - global globalref assign-const-if-global isglobal thismodule + global globalref global-if-global assign-const-if-global isglobal thismodule const atomic null true false ssavalue isdefined toplevel module lambda error gc_preserve_begin gc_preserve_end import using export public inline noinline purity))) diff --git a/test/syntax.jl b/test/syntax.jl index 8125f44cfe693..373a9ad9e0528 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -4116,3 +4116,11 @@ end # Issue #56904 - lambda linearized twice @test (let; try 3; finally try 1; f(() -> x); catch x; end; end; x = 7; end) === 7 @test (let; try 3; finally try 4; finally try 1; f(() -> x); catch x; end; end; end; x = 7; end) === 7 + +# Issue #57546 - explicit function declaration should create new global +module FuncDecl57546 + using Test + @test_nowarn @eval function Any end + @test isa(Any, Function) + @test isempty(methods(Any)) +end From 66085c53759549af750bd643bfc90373e87acfdb Mon Sep 17 00:00:00 2001 From: Diogo Netto <61364108+d-netto@users.noreply.github.com> Date: Sat, 1 Mar 2025 11:30:13 -0300 Subject: [PATCH 17/74] avoid overflow on functionality to print backtrace of safepoint straggler (#57579) In the line of C code: ```C const int64_t timeout = jl_options.timeout_for_safepoint_straggler_s * 1000000000; ``` `jl_options.timeout_for_safepoint_straggler_s` is an `int16_t` and `1000000000` is an `int32_t`. The result of `jl_options.timeout_for_safepoint_straggler_s * 1000000000` will be an `int32_t` which may not be large enough to hold the value of `jl_options.timeout_for_safepoint_straggler_s` after converting to nanoseconds, leading to overflow. (cherry picked from commit 9cafd35eac49206ebe35e5093e58735b7e883b33) --- src/safepoint.c | 2 +- test/threads_exec.jl | 26 ++++++++++++++------------ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/src/safepoint.c b/src/safepoint.c index 66bea539861f8..970c48875d790 100644 --- a/src/safepoint.c +++ b/src/safepoint.c @@ -157,7 +157,7 @@ void jl_gc_wait_for_the_world(jl_ptls_t* gc_all_tls_states, int gc_n_threads) uv_mutex_unlock(&safepoint_lock); } else { - const int64_t timeout = jl_options.timeout_for_safepoint_straggler_s * 1000000000; // convert to nanoseconds + const int64_t timeout = jl_options.timeout_for_safepoint_straggler_s * 1000000000LL; // convert to nanoseconds int ret = 0; uv_mutex_lock(&safepoint_lock); if (!jl_atomic_load_relaxed(&ptls2->gc_state)) { diff --git a/test/threads_exec.jl b/test/threads_exec.jl index 4b8a358d05ca4..629f474f53a38 100644 --- a/test/threads_exec.jl +++ b/test/threads_exec.jl @@ -1630,25 +1630,27 @@ end program = " function main() t = Threads.@spawn begin - ccall(:uv_sleep, Cvoid, (Cuint,), 5000) + ccall(:uv_sleep, Cvoid, (Cuint,), 20_000) end # Force a GC - ccall(:uv_sleep, Cvoid, (Cuint,), 1000) + ccall(:uv_sleep, Cvoid, (Cuint,), 1_000) GC.gc() wait(t) end main() " - tmp_output_filename = tempname() - tmp_output_file = open(tmp_output_filename, "w") - if isnothing(tmp_output_file) - error("Failed to open file $tmp_output_filename") - end - run(pipeline(`$(Base.julia_cmd()) --threads=4 --timeout-for-safepoint-straggler=1 -e $program`, stderr=tmp_output_file)) - # Check whether we printed the straggler's backtrace - @test !isempty(read(tmp_output_filename, String)) - close(tmp_output_file) - rm(tmp_output_filename) + for timeout in ("1", "4", "16") + tmp_output_filename = tempname() + tmp_output_file = open(tmp_output_filename, "w") + if isnothing(tmp_output_file) + error("Failed to open file $tmp_output_filename") + end + run(pipeline(`$(Base.julia_cmd()) --threads=4 --timeout-for-safepoint-straggler=$(timeout) -e $program`, stderr=tmp_output_file)) + # Check whether we printed the straggler's backtrace + @test !isempty(read(tmp_output_filename, String)) + close(tmp_output_file) + rm(tmp_output_filename) + end end end # main testset From e661b5f6693cf92a234c97fb9a8a08d344bfe20d Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Sun, 2 Mar 2025 05:16:47 +0100 Subject: [PATCH 18/74] `Base.Precompilation.ExplicitEnv`: handle type instability better in constructor (#57599) Avoiding the converting `setindex!` should hopefully diminish unwarranted invalidation of user code. (cherry picked from commit 1e03ed6cd5851b30bd140bfc782b9f2a026012a1) --- base/precompilation.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/precompilation.jl b/base/precompilation.jl index 820cf260df71f..5392e119d25a2 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -143,15 +143,16 @@ function ExplicitEnv(envpath::String=Base.active_project()) # Extensions deps_pkg = get(Dict{String, Any}, pkg_info, "extensions")::Dict{String, Any} + deps_pkg_concrete = Dict{String, Vector{String}}() for (ext, triggers) in deps_pkg if triggers isa String triggers = [triggers] else triggers = triggers::Vector{String} end - deps_pkg[ext] = triggers + deps_pkg_concrete[ext] = triggers end - extensions[m_uuid] = deps_pkg + extensions[m_uuid] = deps_pkg_concrete # Determine strategy to find package lookup_strat = begin From e9126d7771f83cff146a3a22b35feeab1f6ca87c Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 1 Mar 2025 23:40:05 -0500 Subject: [PATCH 19/74] cleanup old builtins (#57532) The `_apply_pure` function only has one user, and that user is unsound anyways and should not be used, so replace that with equivalent `_call_in_world_total` call and remove unnecessary definitions. The awkward distinction between `invokelatest` and `_call_latest` has not been relevant (and indeed causes performance issues) since kwfunc was introduced in #47157. (cherry picked from commit 671c6a1636e484f02a10ba2c946e02d6e02bb2d8) --- Compiler/src/abstractinterpretation.jl | 2 +- Compiler/src/ssair/show.jl | 2 +- Compiler/src/verifytrim.jl | 5 +-- base/Base_compiler.jl | 57 ++++++++++++++++++++++++++ base/boot.jl | 7 +++- base/essentials.jl | 57 -------------------------- base/reflection.jl | 4 +- contrib/generate_precompile.jl | 1 - contrib/juliac-buildscript.jl | 13 +----- src/builtin_proto.h | 5 +-- src/builtins.c | 55 ++++++------------------- src/codegen.cpp | 5 +-- src/staticdata.c | 4 +- stdlib/REPL/test/precompilation.jl | 2 +- stdlib/REPL/test/repl.jl | 5 +-- 15 files changed, 91 insertions(+), 133 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index c280b4ee2ad08..734a572313708 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -778,7 +778,7 @@ function edge_matches_sv(interp::AbstractInterpreter, frame::AbsIntState, # If the method defines a recursion relation, give it a chance # to tell us that this recursion is actually ok. if isdefined(method, :recursion_relation) - if Core._apply_pure(method.recursion_relation, Any[method, callee_method2, sig, frame_instance(frame).specTypes]) + if Core._call_in_world_total(get_world_counter(), method.recursion_relation, method, callee_method2, sig, frame_instance(frame).specTypes) return false end end diff --git a/Compiler/src/ssair/show.jl b/Compiler/src/ssair/show.jl index e63d7b5cf640e..0688c02eb6440 100644 --- a/Compiler/src/ssair/show.jl +++ b/Compiler/src/ssair/show.jl @@ -67,7 +67,7 @@ function builtin_call_has_dispatch( return true end end - elseif (f === Core._apply_pure || f === Core._call_in_world || f === Core._call_in_world_total || f === Core._call_latest) + elseif (f === Core.invoke_in_world || f === Core._call_in_world_total || f === Core.invokelatest) # These apply-like builtins are effectively dynamic calls return true end diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index 7706d89483cbf..67728a15177b4 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -153,11 +153,10 @@ function may_dispatch(@nospecialize ftyp) # other builtins (including the IntrinsicFunctions) are good return Core._apply isa ftyp || Core._apply_iterate isa ftyp || - Core._apply_pure isa ftyp || - Core._call_in_world isa ftyp || Core._call_in_world_total isa ftyp || - Core._call_latest isa ftyp || Core.invoke isa ftyp || + Core.invoke_in_world isa ftyp || + Core.invokelatest isa ftyp || Core.finalizer isa ftyp || Core.modifyfield! isa ftyp || Core.modifyglobal! isa ftyp || diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index a1ae14427e754..fc2304f8fce46 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -202,6 +202,63 @@ function Core._hasmethod(@nospecialize(f), @nospecialize(t)) # this function has return Core._hasmethod(tt) end +""" + invokelatest(f, args...; kwargs...) + +Calls `f(args...; kwargs...)`, but guarantees that the most recent method of `f` +will be executed. This is useful in specialized circumstances, +e.g. long-running event loops or callback functions that may +call obsolete versions of a function `f`. +(The drawback is that `invokelatest` is somewhat slower than calling +`f` directly, and the type of the result cannot be inferred by the compiler.) + +!!! compat "Julia 1.9" + Prior to Julia 1.9, this function was not exported, and was called as `Base.invokelatest`. +""" +const invokelatest = Core.invokelatest + +# define invokelatest(f, args...; kwargs...), without kwargs wrapping +# to forward to invokelatest +function Core.kwcall(kwargs::NamedTuple, ::typeof(invokelatest), f, args...) + @inline + return Core.invokelatest(Core.kwcall, kwargs, f, args...) +end +setfield!(typeof(invokelatest).name.mt, :max_args, 2, :monotonic) # invokelatest, f, args... + +""" + invoke_in_world(world, f, args...; kwargs...) + +Call `f(args...; kwargs...)` in a fixed world age, `world`. + +This is useful for infrastructure running in the user's Julia session which is +not part of the user's program. For example, things related to the REPL, editor +support libraries, etc. In these cases it can be useful to prevent unwanted +method invalidation and recompilation latency, and to prevent the user from +breaking supporting infrastructure by mistake. + +The current world age can be queried using [`Base.get_world_counter()`](@ref) +and stored for later use within the lifetime of the current Julia session, or +when serializing and reloading the system image. + +Technically, `invoke_in_world` will prevent any function called by `f` from +being extended by the user during their Julia session. That is, generic +function method tables seen by `f` (and any functions it calls) will be frozen +as they existed at the given `world` age. In a sense, this is like the opposite +of [`invokelatest`](@ref). + +!!! note + It is not valid to store world ages obtained in precompilation for later use. + This is because precompilation generates a "parallel universe" where the + world age refers to system state unrelated to the main Julia session. +""" +const invoke_in_world = Core.invoke_in_world + +function Core.kwcall(kwargs::NamedTuple, ::typeof(invoke_in_world), world::UInt, f, args...) + @inline + return Core.invoke_in_world(world, Core.kwcall, kwargs, f, args...) +end +setfield!(typeof(invoke_in_world).name.mt, :max_args, 3, :monotonic) # invoke_in_world, world, f, args... + # core operations & types include("promotion.jl") include("tuple.jl") diff --git a/base/boot.jl b/base/boot.jl index 93d73679d65ed..b1abe6a65041f 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -1014,8 +1014,11 @@ _parse = nothing _setparser!(parser) = setglobal!(Core, :_parse, parser) -# support for deprecated uses of internal _apply function -_apply(x...) = Core._apply_iterate(Main.Base.iterate, x...) +# support for deprecated uses of builtin functions +_apply(x...) = _apply_iterate(Main.Base.iterate, x...) +_apply_pure(x...) = invoke_in_world_total(typemax_UInt, x...) +const _call_latest = invokelatest +const _call_in_world = invoke_in_world struct Pair{A, B} first::A diff --git a/base/essentials.jl b/base/essentials.jl index 5db7a5f6fb0d9..1e2ec0ef79103 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -1037,63 +1037,6 @@ end Val(x) = Val{x}() -""" - invokelatest(f, args...; kwargs...) - -Calls `f(args...; kwargs...)`, but guarantees that the most recent method of `f` -will be executed. This is useful in specialized circumstances, -e.g. long-running event loops or callback functions that may -call obsolete versions of a function `f`. -(The drawback is that `invokelatest` is somewhat slower than calling -`f` directly, and the type of the result cannot be inferred by the compiler.) - -!!! compat "Julia 1.9" - Prior to Julia 1.9, this function was not exported, and was called as `Base.invokelatest`. -""" -function invokelatest(@nospecialize(f), @nospecialize args...; kwargs...) - @inline - kwargs = merge(NamedTuple(), kwargs) - if isempty(kwargs) - return Core._call_latest(f, args...) - end - return Core._call_latest(Core.kwcall, kwargs, f, args...) -end - -""" - invoke_in_world(world, f, args...; kwargs...) - -Call `f(args...; kwargs...)` in a fixed world age, `world`. - -This is useful for infrastructure running in the user's Julia session which is -not part of the user's program. For example, things related to the REPL, editor -support libraries, etc. In these cases it can be useful to prevent unwanted -method invalidation and recompilation latency, and to prevent the user from -breaking supporting infrastructure by mistake. - -The current world age can be queried using [`Base.get_world_counter()`](@ref) -and stored for later use within the lifetime of the current Julia session, or -when serializing and reloading the system image. - -Technically, `invoke_in_world` will prevent any function called by `f` from -being extended by the user during their Julia session. That is, generic -function method tables seen by `f` (and any functions it calls) will be frozen -as they existed at the given `world` age. In a sense, this is like the opposite -of [`invokelatest`](@ref). - -!!! note - It is not valid to store world ages obtained in precompilation for later use. - This is because precompilation generates a "parallel universe" where the - world age refers to system state unrelated to the main Julia session. -""" -function invoke_in_world(world::UInt, @nospecialize(f), @nospecialize args...; kwargs...) - @inline - kwargs = Base.merge(NamedTuple(), kwargs) - if isempty(kwargs) - return Core._call_in_world(world, f, args...) - end - return Core._call_in_world(world, Core.kwcall, kwargs, f, args...) -end - """ inferencebarrier(x) diff --git a/base/reflection.jl b/base/reflection.jl index c98f6244cc89f..50b4413f01b59 100644 --- a/base/reflection.jl +++ b/base/reflection.jl @@ -1283,9 +1283,9 @@ function invokelatest_gr(gr::GlobalRef, @nospecialize args...; kwargs...) @inline kwargs = merge(NamedTuple(), kwargs) if isempty(kwargs) - return Core._call_latest(apply_gr, gr, args...) + return invokelatest(apply_gr, gr, args...) end - return Core._call_latest(apply_gr_kw, kwargs, gr, args...) + return invokelatest(apply_gr_kw, kwargs, gr, args...) end """ diff --git a/contrib/generate_precompile.jl b/contrib/generate_precompile.jl index f12e94e22fca5..ca95b4fa2fcdb 100644 --- a/contrib/generate_precompile.jl +++ b/contrib/generate_precompile.jl @@ -46,7 +46,6 @@ precompile(Tuple{typeof(Base.Threads.atomic_sub!), Base.Threads.Atomic{Int}, Int precompile(Tuple{Type{Base.Val{x} where x}, Module}) precompile(Tuple{Type{NamedTuple{(:honor_overrides,), T} where T<:Tuple}, Tuple{Bool}}) precompile(Tuple{typeof(Base.unique!), Array{String, 1}}) -precompile(Tuple{typeof(Base.invokelatest), Any}) precompile(Tuple{typeof(Base.vcat), Array{String, 1}, Array{String, 1}}) # Pkg loading diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index 7d25e85ff4c5c..459922ab1ccb1 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -38,17 +38,8 @@ end init_active_project() = nothing disable_library_threading() = nothing start_profile_listener() = nothing - @inline function invokelatest(f::F, args...; kwargs...) where F - return f(args...; kwargs...) - end - @inline function invokelatest_gr(gr::GlobalRef, @nospecialize args...; kwargs...) - @inline - kwargs = merge(NamedTuple(), kwargs) - if isempty(kwargs) - return apply_gr(gr, args...) - end - return apply_gr_kw(kwargs, gr, args...) - end + invokelatest_trimmed(f, args...; kwargs...) = f(args...; kwargs...) + const invokelatest = invokelatest_trimmed function sprint(f::F, args::Vararg{Any,N}; context=nothing, sizehint::Integer=0) where {F<:Function,N} s = IOBuffer(sizehint=sizehint) if context isa Tuple diff --git a/src/builtin_proto.h b/src/builtin_proto.h index a543aa895fb97..c82ec77414129 100644 --- a/src/builtin_proto.h +++ b/src/builtin_proto.h @@ -22,10 +22,9 @@ extern "C" { #endif DECLARE_BUILTIN(_apply_iterate); -DECLARE_BUILTIN(_apply_pure); -DECLARE_BUILTIN(_call_in_world); +DECLARE_BUILTIN(invoke_in_world); DECLARE_BUILTIN(_call_in_world_total); -DECLARE_BUILTIN(_call_latest); +DECLARE_BUILTIN(invokelatest); DECLARE_BUILTIN(_compute_sparams); DECLARE_BUILTIN(_expr); DECLARE_BUILTIN(_svec_ref); diff --git a/src/builtins.c b/src/builtins.c index 063b191510bfd..243d14f34d3bd 100644 --- a/src/builtins.c +++ b/src/builtins.c @@ -637,9 +637,14 @@ static jl_value_t *jl_arrayref(jl_array_t *a, size_t i) return jl_memoryrefget(jl_memoryrefindex(a->ref, i), 0); } -static jl_value_t *do_apply(jl_value_t **args, uint32_t nargs, jl_value_t *iterate) +JL_CALLABLE(jl_f__apply_iterate) { - jl_function_t *f = args[0]; + JL_NARGSV(_apply_iterate, 2); + jl_function_t *iterate = args[0]; + jl_function_t *f = args[1]; + assert(iterate); + args += 1; + nargs -= 1; if (nargs == 2) { // some common simple cases if (f == jl_builtin_svec) { @@ -692,9 +697,6 @@ static jl_value_t *do_apply(jl_value_t **args, uint32_t nargs, jl_value_t *itera extra += 1; } } - if (extra && iterate == NULL) { - jl_undefined_var_error(jl_symbol("iterate"), NULL); - } // allocate space for the argument array and gc roots for it // based on our previous estimates // use the stack if we have a good estimate that it is small @@ -841,40 +843,8 @@ static jl_value_t *do_apply(jl_value_t **args, uint32_t nargs, jl_value_t *itera return result; } -JL_CALLABLE(jl_f__apply_iterate) -{ - JL_NARGSV(_apply_iterate, 2); - return do_apply(args + 1, nargs - 1, args[0]); -} - -// this is like `_apply`, but with quasi-exact checks to make sure it is pure -JL_CALLABLE(jl_f__apply_pure) -{ - jl_task_t *ct = jl_current_task; - int last_in = ct->ptls->in_pure_callback; - jl_value_t *ret = NULL; - JL_TRY { - ct->ptls->in_pure_callback = 1; - // because this function was declared pure, - // we should be allowed to run it in any world - // so we run it in the newest world; - // because, why not :) - // and `promote` works better this way - size_t last_age = ct->world_age; - ct->world_age = jl_atomic_load_acquire(&jl_world_counter); - ret = do_apply(args, nargs, NULL); - ct->world_age = last_age; - ct->ptls->in_pure_callback = last_in; - } - JL_CATCH { - ct->ptls->in_pure_callback = last_in; - jl_rethrow(); - } - return ret; -} - // this is like a regular call, but always runs in the newest world -JL_CALLABLE(jl_f__call_latest) +JL_CALLABLE(jl_f_invokelatest) { jl_task_t *ct = jl_current_task; size_t last_age = ct->world_age; @@ -885,9 +855,9 @@ JL_CALLABLE(jl_f__call_latest) return ret; } -// Like call_in_world, but runs in the specified world. +// Like invokelatest, but runs in the specified world. // If world > jl_atomic_load_acquire(&jl_world_counter), run in the latest world. -JL_CALLABLE(jl_f__call_in_world) +JL_CALLABLE(jl_f_invoke_in_world) { JL_NARGSV(_apply_in_world, 2); jl_task_t *ct = jl_current_task; @@ -2539,9 +2509,8 @@ void jl_init_primitives(void) JL_GC_DISABLED jl_builtin__apply_iterate = add_builtin_func("_apply_iterate", jl_f__apply_iterate); jl_builtin__expr = add_builtin_func("_expr", jl_f__expr); jl_builtin_svec = add_builtin_func("svec", jl_f_svec); - add_builtin_func("_apply_pure", jl_f__apply_pure); - add_builtin_func("_call_latest", jl_f__call_latest); - add_builtin_func("_call_in_world", jl_f__call_in_world); + add_builtin_func("invokelatest", jl_f_invokelatest); + add_builtin_func("invoke_in_world", jl_f_invoke_in_world); add_builtin_func("_call_in_world_total", jl_f__call_in_world_total); add_builtin_func("_typevar", jl_f__typevar); add_builtin_func("_structtype", jl_f__structtype); diff --git a/src/codegen.cpp b/src/codegen.cpp index 17d005e09200e..2d0be5f782281 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1595,9 +1595,8 @@ static const auto &builtin_func_map() { { jl_f_typeassert_addr, new JuliaFunction<>{XSTR(jl_f_typeassert), get_func_sig, get_func_attrs} }, { jl_f_ifelse_addr, new JuliaFunction<>{XSTR(jl_f_ifelse), get_func_sig, get_func_attrs} }, { jl_f__apply_iterate_addr, new JuliaFunction<>{XSTR(jl_f__apply_iterate), get_func_sig, get_func_attrs} }, - { jl_f__apply_pure_addr, new JuliaFunction<>{XSTR(jl_f__apply_pure), get_func_sig, get_func_attrs} }, - { jl_f__call_latest_addr, new JuliaFunction<>{XSTR(jl_f__call_latest), get_func_sig, get_func_attrs} }, - { jl_f__call_in_world_addr, new JuliaFunction<>{XSTR(jl_f__call_in_world), get_func_sig, get_func_attrs} }, + { jl_f_invokelatest_addr, new JuliaFunction<>{XSTR(jl_f_invokelatest), get_func_sig, get_func_attrs} }, + { jl_f_invoke_in_world_addr, new JuliaFunction<>{XSTR(jl_f_invoke_in_world), get_func_sig, get_func_attrs} }, { jl_f__call_in_world_total_addr, new JuliaFunction<>{XSTR(jl_f__call_in_world_total), get_func_sig, get_func_attrs} }, { jl_f_throw_addr, new JuliaFunction<>{XSTR(jl_f_throw), get_func_sig, get_func_attrs} }, { jl_f_throw_methoderror_addr, new JuliaFunction<>{XSTR(jl_f_throw_methoderror), get_func_sig, get_func_attrs} }, diff --git a/src/staticdata.c b/src/staticdata.c index 666bf2d8ccd29..7dc23ed4c9449 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -507,8 +507,8 @@ static htable_t bits_replace; // This is a manually constructed dual of the fvars array, which would be produced by codegen for Julia code, for C. static const jl_fptr_args_t id_to_fptrs[] = { &jl_f_throw, &jl_f_throw_methoderror, &jl_f_is, &jl_f_typeof, &jl_f_issubtype, &jl_f_isa, - &jl_f_typeassert, &jl_f__apply_iterate, &jl_f__apply_pure, - &jl_f__call_latest, &jl_f__call_in_world, &jl_f__call_in_world_total, &jl_f_isdefined, &jl_f_isdefinedglobal, + &jl_f_typeassert, &jl_f__apply_iterate, + &jl_f_invokelatest, &jl_f_invoke_in_world, &jl_f__call_in_world_total, &jl_f_isdefined, &jl_f_isdefinedglobal, &jl_f_tuple, &jl_f_svec, &jl_f_intrinsic_call, &jl_f_getfield, &jl_f_setfield, &jl_f_swapfield, &jl_f_modifyfield, &jl_f_setfieldonce, &jl_f_replacefield, &jl_f_fieldtype, &jl_f_nfields, &jl_f_apply_type, &jl_f_memorynew, diff --git a/stdlib/REPL/test/precompilation.jl b/stdlib/REPL/test/precompilation.jl index 01a062644596c..7efcf0b5e8282 100644 --- a/stdlib/REPL/test/precompilation.jl +++ b/stdlib/REPL/test/precompilation.jl @@ -33,7 +33,7 @@ if !Sys.iswindows() # given this test checks that startup is snappy, it's best to add workloads to # contrib/generate_precompile.jl rather than increase this number. But if that's not # possible, it'd be helpful to add a comment with the statement and a reason below - expected_precompiles = 1 + expected_precompiles = 0 n_precompiles = count(r"precompile\(", tracecompile_out) diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index 69f0cb846d8d7..a0bfa9220f97b 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -244,9 +244,8 @@ fake_repl(options = REPL.Options(confirm_exit=false,hascolor=true)) do stdin_wri @test occursin("shell> ", s) # check for the echo of the prompt @test occursin("'", s) # check for the echo of the input s = readuntil(stdout_read, "\n\n") - @test(startswith(s, "\e[0mERROR: unterminated single quote\nStacktrace:\n [1] ") || - startswith(s, "\e[0m\e[1m\e[91mERROR: \e[39m\e[22m\e[91munterminated single quote\e[39m\nStacktrace:\n [1] "), - skip = Sys.iswindows() && Sys.WORD_SIZE == 32) + @test(startswith(s, "\e[0mERROR: unterminated single quote\nStacktrace:\n [1] ") || + startswith(s, "\e[0m\e[1m\e[91mERROR: \e[39m\e[22m\e[91munterminated single quote\e[39m\nStacktrace:\n [1] ")) write(stdin_write, "\b") wait(t) end From 38644b332ff0e722c3a9d30c5e9b1eea53003aa6 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 3 Mar 2025 09:41:36 +0100 Subject: [PATCH 20/74] strings: type assert in the generic `nextind`, `prevind` methods (#57608) The type assertions are valid according to the doc strings of these functions in the case of `AbstractString`. Should prevent some invalidation on loading user code. Fixes #57605 (cherry picked from commit 6c9c336d2acc71ed740b0cc470c0b76bbbda136c) --- base/strings/basic.jl | 12 ++++++------ test/strings/basic.jl | 5 +++++ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/base/strings/basic.jl b/base/strings/basic.jl index 3f26aed736b8d..c40deb0656ced 100644 --- a/base/strings/basic.jl +++ b/base/strings/basic.jl @@ -513,11 +513,11 @@ prevind(s::AbstractString, i::Int) = prevind(s, i, 1) function prevind(s::AbstractString, i::Int, n::Int) n < 0 && throw(ArgumentError("n cannot be negative: $n")) - z = ncodeunits(s) + 1 + z = ncodeunits(s)::Int + 1 @boundscheck 0 < i ≤ z || throw(BoundsError(s, i)) - n == 0 && return thisind(s, i) == i ? i : string_index_err(s, i) + n == 0 && return thisind(s, i)::Int == i ? i : string_index_err(s, i) while n > 0 && 1 < i - @inbounds n -= isvalid(s, i -= 1) + @inbounds n -= isvalid(s, i -= 1)::Bool end return i - n end @@ -572,11 +572,11 @@ nextind(s::AbstractString, i::Int) = nextind(s, i, 1) function nextind(s::AbstractString, i::Int, n::Int) n < 0 && throw(ArgumentError("n cannot be negative: $n")) - z = ncodeunits(s) + z = ncodeunits(s)::Int @boundscheck 0 ≤ i ≤ z || throw(BoundsError(s, i)) - n == 0 && return thisind(s, i) == i ? i : string_index_err(s, i) + n == 0 && return thisind(s, i)::Int == i ? i : string_index_err(s, i) while n > 0 && i < z - @inbounds n -= isvalid(s, i += 1) + @inbounds n -= isvalid(s, i += 1)::Bool end return i + n end diff --git a/test/strings/basic.jl b/test/strings/basic.jl index f90ce8c697ed8..c3e0bcc501070 100644 --- a/test/strings/basic.jl +++ b/test/strings/basic.jl @@ -877,6 +877,11 @@ end end end end + + @testset "return type infers to `Int`" begin + @test Int === Base.infer_return_type(prevind, Tuple{AbstractString, Vararg}) + @test Int === Base.infer_return_type(nextind, Tuple{AbstractString, Vararg}) + end end @testset "first and last" begin From 4942b1cbd59db41e75abdcc875a78faf2af88ddd Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 3 Mar 2025 09:42:50 +0100 Subject: [PATCH 21/74] `AnnotatedString`: add concrete type asserts to `isvalid`, `ncodeunits` (#57607) The type assertions are valid according to the doc strings of these two functions in the case of `AbstractString`. Should prevent some invalidation on loading user code. Fixes #57606 (cherry picked from commit c4eeabf2f64ddfc133b02be04f8b6557d5f722e9) --- base/strings/annotated.jl | 4 ++-- test/strings/annotated.jl | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/base/strings/annotated.jl b/base/strings/annotated.jl index 814ee2afa9d55..0dcac0bf2de3b 100644 --- a/base/strings/annotated.jl +++ b/base/strings/annotated.jl @@ -147,11 +147,11 @@ promote_rule(::Type{<:AnnotatedString}, ::Type{<:AbstractString}) = AnnotatedStr ## AbstractString interface ## -ncodeunits(s::AnnotatedString) = ncodeunits(s.string) +ncodeunits(s::AnnotatedString) = ncodeunits(s.string)::Int codeunits(s::AnnotatedString) = codeunits(s.string) codeunit(s::AnnotatedString) = codeunit(s.string) codeunit(s::AnnotatedString, i::Integer) = codeunit(s.string, i) -isvalid(s::AnnotatedString, i::Integer) = isvalid(s.string, i) +isvalid(s::AnnotatedString, i::Integer) = isvalid(s.string, i)::Bool @propagate_inbounds iterate(s::AnnotatedString, i::Integer=firstindex(s)) = if i <= lastindex(s.string); (s[i], nextind(s, i)) end eltype(::Type{<:AnnotatedString{S}}) where {S} = AnnotatedChar{eltype(S)} diff --git a/test/strings/annotated.jl b/test/strings/annotated.jl index dbb81cb48acbc..8658c1b52a2ab 100644 --- a/test/strings/annotated.jl +++ b/test/strings/annotated.jl @@ -74,6 +74,9 @@ a = Base.AnnotatedString("hello", [(1:5, :label, 1)]) @test first(a) == Base.AnnotatedChar('h', [(:label, 1)]) + + @test Bool === Base.infer_return_type(isvalid, Tuple{Base.AnnotatedString, Vararg}) + @test Int === Base.infer_return_type(ncodeunits, Tuple{Base.AnnotatedString}) end @testset "AnnotatedChar" begin From 4eaae59d8a6ffed4945242f9a37bc2bed6d97718 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 3 Mar 2025 15:22:23 +0100 Subject: [PATCH 22/74] `Base`: `power_by_squaring`: don't require `one` (#57590) * Follows up on/partially reverts #55634 * Reopens #53504, perhaps it's OK to require some types to implement their own `^` for good performance * Fixes #57390, a regression (cherry picked from commit a984a213b6e5b6b407c7878e413b680bfd15f2a2) --- base/intfuncs.jl | 2 +- test/math.jl | 9 ++++ test/testhelpers/EvenIntegers.jl | 87 ++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 test/testhelpers/EvenIntegers.jl diff --git a/base/intfuncs.jl b/base/intfuncs.jl index bb0f454d9c574..1bbab0224493e 100644 --- a/base/intfuncs.jl +++ b/base/intfuncs.jl @@ -336,7 +336,7 @@ end # ^ for any x supporting * function to_power_type(x::Number) - T = promote_type(typeof(x), typeof(one(x)), typeof(x*x)) + T = promote_type(typeof(x), typeof(x*x)) convert(T, x) end to_power_type(x) = oftype(x*x, x) diff --git a/test/math.jl b/test/math.jl index d9cfd411124ed..8f5ada013557c 100644 --- a/test/math.jl +++ b/test/math.jl @@ -1,5 +1,8 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license +include("testhelpers/EvenIntegers.jl") +using .EvenIntegers + using Random using LinearAlgebra using Base.Experimental: @force_compile @@ -1538,6 +1541,12 @@ end @test all((t -> ===(t...)), zip(x^y, p[y + 1])) end end + + @testset "rng exponentiation, issue #57590" begin + @test EvenInteger(16) === @inferred EvenInteger(2)^4 + @test EvenInteger(16) === @inferred EvenInteger(2)^Int(4) # avoid `literal_pow` + @test EvenInteger(16) === @inferred EvenInteger(2)^EvenInteger(4) + end end # Test that sqrt behaves correctly and doesn't exhibit fp80 double rounding. diff --git a/test/testhelpers/EvenIntegers.jl b/test/testhelpers/EvenIntegers.jl new file mode 100644 index 0000000000000..2926d1ce65109 --- /dev/null +++ b/test/testhelpers/EvenIntegers.jl @@ -0,0 +1,87 @@ +""" +The even integers, an example of set with an additive identity and closed under +addition and multiplication, but lacking a multiplicative identity, a +[*rng*](https://en.wikipedia.org/wiki/Rng_(algebra)). +""" +module EvenIntegers + export EvenInteger + + struct EvenInteger{T <: Integer} <: Integer + x::T + function EvenInteger(x::Integer) + if isodd(x) + throw(ArgumentError("can't convert odd integer to even integer")) + end + new{typeof(x)}(x) + end + end + function EvenInteger(x::EvenInteger) + x + end + function EvenInteger{T}(x::EvenInteger{T}) where {T <: Integer} + x + end + function EvenInteger{T}(x::T) where {T <: Integer} + EvenInteger(x) + end + function EvenInteger{T}(x::Integer) where {T <: Integer} + throw(ArgumentError("not implemented")) + end + function Base.Int(n::EvenInteger) + Int(n.x) + end + function Base.iseven(::EvenInteger) + true + end + function Base.isodd(::EvenInteger) + false + end + function Base.iszero(n::EvenInteger) + iszero(n.x) + end + function Base.isone(::EvenInteger) + false + end + function Base.zero(n::EvenInteger) + EvenInteger(zero(n.x)) + end + function Base.zero(::Type{EvenInteger{T}}) where {T <: Integer} + EvenInteger(zero(T)) + end + function Base.:(==)(l::EvenInteger, r::EvenInteger) + l.x == r.x + end + function Base.:(<)(l::EvenInteger, r::EvenInteger) + l.x < r.x + end + function Base.promote_rule(::Type{EvenInteger{L}}, ::Type{EvenInteger{R}}) where {L <: Integer, R <: Integer} + EvenInteger{promote_type(L, R)} + end + function Base.promote_rule(::Type{EvenInteger{L}}, ::Type{R}) where {L <: Integer, R <: Integer} + promote_type(L, R) + end + function Base.:(+)(l::EvenInteger, r::EvenInteger) + EvenInteger(l.x + r.x) + end + function Base.:(*)(l::EvenInteger, r::EvenInteger) + EvenInteger(l.x * r.x) + end + function Base.:(-)(n::EvenInteger) + EvenInteger(-n.x) + end + function Base.:(-)(l::EvenInteger, r::EvenInteger) + l + (-r) + end + function right_shift(l::EvenInteger, r::Integer) + l.x >> r + end + function Base.:(>>)(l::EvenInteger, r::Integer) + right_shift(l, r) + end + function Base.:(>>)(l::EvenInteger, r::Int) # resolve dispatch ambiguity + right_shift(l, r) + end + function Base.trailing_zeros(n::EvenInteger) + trailing_zeros(n.x) + end +end From ad929027723553e3a3145d3ad913a720e0298945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1ll=20Haraldsson?= Date: Tue, 4 Mar 2025 09:30:48 +0000 Subject: [PATCH 23/74] Precompile Markdown parsing AND printing (#55752) --------- Co-authored-by: Kristoffer Co-authored-by: Ian Butterworth (cherry picked from commit 0dd78f72df2003f0ace3cc319403e01e3d3c2b09) --- stdlib/Markdown/src/Markdown.jl | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/stdlib/Markdown/src/Markdown.jl b/stdlib/Markdown/src/Markdown.jl index 8d79cc93d6171..723eb6ca68482 100644 --- a/stdlib/Markdown/src/Markdown.jl +++ b/stdlib/Markdown/src/Markdown.jl @@ -138,12 +138,16 @@ catdoc(md::MD...) = MD(md...) if Base.generating_output() # workload to reduce latency - md""" + show(devnull, MIME("text/plain"), md""" # H1 ## H2 ### H3 + #### H4 + ##### H5 + ###### H6 **bold text** *italicized text* + ***bold and italicized text*** > blockquote 1. First item 2. Second item @@ -151,10 +155,18 @@ if Base.generating_output() - First item - Second item - Third item + - Indented item `code` Horizontal Rule --- - """ + **[Duck Duck Go](https://duckduckgo.com)** + + + ![The San Juan Mountains are beautiful!](/assets/images/san-juan-mountains.jpg "San Juan Mountains") + + H~2~O + X^2^ + """) end end From b3c81e95603ff0e47b2bb06b6b6b037d7bb69cda Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Thu, 27 Feb 2025 22:33:51 -0500 Subject: [PATCH 24/74] bpart: Also partition ->deprecated (#57449) This repeats the exercise in #57405. This is required for correctness, because the ->deprecated flag also affects `using` resolution (since it makes the tagged binding weaker for `using` purposes). That said, in general our `->deprecated` semantics have been somewhat underspecified with lots of `XXX` comments in the surrounding code. This tries to define the semantics to give a depwarn on *access* (read or write) when: 1. Either the binding itself is deprecated; or 2. The implicit imports pass through a deprecated binding. However, we do not give depwarns on access to bindings that were explicitly imported (although we do give those warnings on the import) - the reasoning being that it's the import that needs to be adjusted not the access. Additionally, this PR moves into the direction of making the depwarn a semantic part of the global access, by adjusting codegen and inference appropriately. (cherry picked from commit 1a3cbb1de6b0b01ef005d74ecc8da7d87b7e8097) --- Compiler/src/Compiler.jl | 1 + Compiler/src/abstractinterpretation.jl | 14 +- Compiler/test/irpasses.jl | 5 +- base/runtime_internals.jl | 5 + base/show.jl | 17 ++- src/codegen.cpp | 119 +++++++++-------- src/jl_exported_funcs.inc | 1 - src/julia.h | 16 ++- src/julia_internal.h | 38 +++++- src/module.c | 177 +++++++++++++++++-------- src/staticdata.c | 4 +- src/toplevel.c | 2 +- test/cmdlineargs.jl | 2 +- 13 files changed, 263 insertions(+), 138 deletions(-) diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index fc2ac491f49d6..490d988565ffd 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -50,6 +50,7 @@ using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstanc using Base using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospecializeinfer, BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST, BINDING_KIND_DECLARED, + BINDING_FLAG_DEPWARN, Base, BitVector, Bottom, Callable, DataTypeFieldDesc, EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES, OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME, diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 734a572313708..b5882deb081df 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2385,7 +2385,7 @@ function abstract_throw_methoderror(interp::AbstractInterpreter, argtypes::Vecto return Future(CallMeta(Union{}, exct, EFFECTS_THROWS, NoCallInfo())) end -const generic_getglobal_effects = Effects(EFFECTS_THROWS, consistent=ALWAYS_FALSE, inaccessiblememonly=ALWAYS_FALSE) +const generic_getglobal_effects = Effects(EFFECTS_THROWS, effect_free=ALWAYS_FALSE, consistent=ALWAYS_FALSE, inaccessiblememonly=ALWAYS_FALSE) #= effect_free for depwarn =# const generic_getglobal_exct = Union{ArgumentError, TypeError, ConcurrencyViolationError, UndefVarError} function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, @nospecialize(M), @nospecialize(s)) ⊑ = partialorder(typeinf_lattice(interp)) @@ -3501,13 +3501,15 @@ end function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Core.BindingPartition) kind = binding_kind(partition) + isdepwarn = (partition.kind & BINDING_FLAG_DEPWARN) != 0 + local_getglobal_effects = Effects(generic_getglobal_effects, effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE) if is_some_guard(kind) || kind == BINDING_KIND_UNDEF_CONST if InferenceParams(interp).assume_bindings_static return RTEffects(Union{}, UndefVarError, EFFECTS_THROWS) else # We do not currently assume an invalidation for guard -> defined transitions # return RTEffects(Union{}, UndefVarError, EFFECTS_THROWS) - return RTEffects(Any, UndefVarError, generic_getglobal_effects) + return RTEffects(Any, UndefVarError, local_getglobal_effects) end end @@ -3515,10 +3517,12 @@ function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Co if kind == BINDING_KIND_BACKDATED_CONST # Infer this as guard. We do not want a later const definition to retroactively improve # inference results in an earlier world. - return RTEffects(Any, UndefVarError, generic_getglobal_effects) + return RTEffects(Any, UndefVarError, local_getglobal_effects) end rt = Const(partition_restriction(partition)) - return RTEffects(rt, Union{}, Effects(EFFECTS_TOTAL, inaccessiblememonly=is_mutation_free_argtype(rt) ? ALWAYS_TRUE : ALWAYS_FALSE)) + return RTEffects(rt, Union{}, Effects(EFFECTS_TOTAL, + inaccessiblememonly=is_mutation_free_argtype(rt) ? ALWAYS_TRUE : ALWAYS_FALSE, + effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE)) end if kind == BINDING_KIND_DECLARED @@ -3526,7 +3530,7 @@ function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Co else rt = partition_restriction(partition) end - return RTEffects(rt, UndefVarError, generic_getglobal_effects) + return RTEffects(rt, UndefVarError, local_getglobal_effects) end function abstract_eval_globalref(interp::AbstractInterpreter, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) diff --git a/Compiler/test/irpasses.jl b/Compiler/test/irpasses.jl index 0d0b0e4daf83e..4593aa3223902 100644 --- a/Compiler/test/irpasses.jl +++ b/Compiler/test/irpasses.jl @@ -1179,7 +1179,10 @@ let ci = code_typed(foo_cfg_empty, Tuple{Bool}, optimize=true)[1][1] end @test Compiler.is_effect_free(Base.infer_effects(getfield, (Complex{Int}, Symbol))) -@test Compiler.is_effect_free(Base.infer_effects(getglobal, (Module, Symbol))) + +# We consider a potential deprecatio warning an effect, so for completely unkown getglobal, +# we taint the effect_free bit. +@test !Compiler.is_effect_free(Base.infer_effects(getglobal, (Module, Symbol))) # Test that UseRefIterator gets SROA'd inside of new_to_regular (#44557) # expression and new_to_regular offset are arbitrary here, we just want to see the UseRefIterator erased diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 2c7bfa70055ae..cce71bdecffb4 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -210,6 +210,11 @@ const BINDING_KIND_UNDEF_CONST = 0x9 const BINDING_KIND_BACKDATED_CONST = 0xa const BINDING_FLAG_EXPORTED = 0x10 +const BINDING_FLAG_DEPRECATED = 0x20 +const BINDING_FLAG_DEPWARN = 0x40 + +const BINDING_KIND_MASK = 0x0f +const BINDING_FLAG_MASK = 0xf0 is_defined_const_binding(kind::UInt8) = (kind == BINDING_KIND_CONST || kind == BINDING_KIND_CONST_IMPORT || kind == BINDING_KIND_BACKDATED_CONST) is_some_const_binding(kind::UInt8) = (is_defined_const_binding(kind) || kind == BINDING_KIND_UNDEF_CONST) diff --git a/base/show.jl b/base/show.jl index 8c9f18c2e01fe..f1336687fbef5 100644 --- a/base/show.jl +++ b/base/show.jl @@ -3370,8 +3370,21 @@ function print_partition(io::IO, partition::Core.BindingPartition) else print(io, max_world) end - if (partition.kind & BINDING_FLAG_EXPORTED) != 0 - print(io, " [exported]") + if (partition.kind & BINDING_FLAG_MASK) != 0 + first = false + print(io, " [") + if (partition.kind & BINDING_FLAG_EXPORTED) != 0 + print(io, "exported") + end + if (partition.kind & BINDING_FLAG_DEPRECATED) != 0 + first ? (first = false) : print(io, ",") + print(io, "deprecated") + end + if (partition.kind & BINDING_FLAG_DEPWARN) != 0 + first ? (first = false) : print(io, ",") + print(io, "depwarn") + end + print(io, "]") end print(io, " - ") kind = binding_kind(partition) diff --git a/src/codegen.cpp b/src/codegen.cpp index 2d0be5f782281..58c59496ccc2f 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -921,13 +921,12 @@ static const auto jldeclareglobal_func = new JuliaFunction<>{ {T_pjlvalue, T_pjlvalue, T_prjlvalue, getInt32Ty(C)}, false); }, nullptr, }; -static const auto jlgetbindingorerror_func = new JuliaFunction<>{ - XSTR(jl_get_binding_or_error), +static const auto jldepcheck_func = new JuliaFunction<>{ + XSTR(jl_binding_deprecation_check), [](LLVMContext &C) { auto T_pjlvalue = JuliaType::get_pjlvalue_ty(C); - return FunctionType::get(T_pjlvalue, - {T_pjlvalue, T_pjlvalue}, false); - }, + return FunctionType::get(getVoidTy(C), + {T_pjlvalue}, false); }, nullptr, }; static const auto jlcheckbpwritable_func = new JuliaFunction<>{ @@ -2904,20 +2903,6 @@ static void mallocVisitLine(jl_codectx_t &ctx, StringRef filename, int line, Val // --- constant determination --- -static void show_source_loc(jl_codectx_t &ctx, JL_STREAM *out) -{ - jl_printf(out, "in %s at %s", ctx.name, ctx.file.str().c_str()); -} - -static void cg_bdw(jl_codectx_t &ctx, jl_sym_t *var, jl_binding_t *b) -{ - jl_binding_deprecation_warning(ctx.module, var, b); - if (b->deprecated == 1 && jl_options.depwarn) { - show_source_loc(ctx, JL_STDERR); - jl_printf(JL_STDERR, "\n"); - } -} - static jl_value_t *static_apply_type(jl_codectx_t &ctx, ArrayRef args, size_t nargs) { assert(nargs > 1); @@ -2942,6 +2927,12 @@ static jl_value_t *static_apply_type(jl_codectx_t &ctx, ArrayRef arg return result; } +static void emit_depwarn_check(jl_codectx_t &ctx, jl_binding_t *b) +{ + Value *bp = julia_binding_gv(ctx, b); + ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); +} + // try to statically evaluate, NULL if not possible. note that this may allocate, and as // such the resulting value should not be embedded directly in the generated code. static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) @@ -2950,9 +2941,13 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) jl_sym_t *sym = (jl_sym_t*)ex; jl_binding_t *bnd = jl_get_module_binding(ctx.module, sym, 0); jl_binding_partition_t *bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); - jl_walk_binding_inplace_all(&bnd, &bpart, ctx.min_world, ctx.max_world); - if (bpart && jl_bkind_is_some_constant(jl_binding_kind(bpart))) + int possibly_deprecated = 0; + jl_walk_binding_inplace_all(&bnd, &bpart, &possibly_deprecated, ctx.min_world, ctx.max_world); + if (bpart && jl_bkind_is_some_constant(jl_binding_kind(bpart))) { + if (possibly_deprecated) + emit_depwarn_check(ctx, bnd); return bpart->restriction; + } return NULL; } if (jl_is_slotnumber(ex) || jl_is_argument(ex)) @@ -2975,13 +2970,14 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) s = jl_globalref_name(ex); jl_binding_t *bnd = jl_get_module_binding(jl_globalref_mod(ex), s, 0); jl_binding_partition_t *bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); - jl_walk_binding_inplace_all(&bnd, &bpart, ctx.min_world, ctx.max_world); + int possibly_deprecated = 0; + jl_walk_binding_inplace_all(&bnd, &bpart, &possibly_deprecated, ctx.min_world, ctx.max_world); jl_value_t *v = NULL; if (bpart && jl_bkind_is_some_constant(jl_binding_kind(bpart))) v = bpart->restriction; if (v) { - if (bnd->deprecated) - cg_bdw(ctx, s, bnd); + if (possibly_deprecated) + emit_depwarn_check(ctx, bnd); return v; } return NULL; @@ -3002,13 +2998,14 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) if (s && jl_is_symbol(s)) { jl_binding_t *bnd = jl_get_module_binding(m, s, 0); jl_binding_partition_t *bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); - jl_walk_binding_inplace_all(&bnd, &bpart, ctx.min_world, ctx.max_world); + int possibly_deprecated = 0; + jl_walk_binding_inplace_all(&bnd, &bpart, &possibly_deprecated, ctx.min_world, ctx.max_world); jl_value_t *v = NULL; if (bpart && jl_bkind_is_some_constant(jl_binding_kind(bpart))) v = bpart->restriction; if (v) { - if (bnd->deprecated) - cg_bdw(ctx, s, bnd); + if (possibly_deprecated) + emit_depwarn_check(ctx, bnd); return v; } } @@ -3254,48 +3251,47 @@ static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t * if (!bpart) { return emit_globalref_runtime(ctx, bnd, mod, name); } - // bpart was updated in place - this will change with full partition - if (jl_bkind_is_some_guard(jl_binding_kind(bpart))) { - // Redo the lookup at runtime - return emit_globalref_runtime(ctx, bnd, mod, name); - } else { - while (true) { - if (!bpart) - break; - if (!jl_bkind_is_some_import(jl_binding_kind(bpart))) - break; - if (bnd->deprecated) { - cg_bdw(ctx, name, bnd); - } - bnd = (jl_binding_t*)bpart->restriction; - bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); - if (!bpart) - break; - } - if (bpart) { - enum jl_partition_kind kind = jl_binding_kind(bpart); - if (jl_bkind_is_some_constant(kind) && kind != BINDING_KIND_BACKDATED_CONST) { - jl_value_t *constval = bpart->restriction; - if (!constval) { - undef_var_error_ifnot(ctx, ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 0), name, (jl_value_t*)mod); - return jl_cgval_t(); - } - return mark_julia_const(ctx, constval); + int possibly_deprecated = 0; + int saw_explicit = 0; + while (bpart) { + if (!saw_explicit && (bpart->kind & BINDING_FLAG_DEPWARN)) + possibly_deprecated = 1; + enum jl_partition_kind kind = jl_binding_kind(bpart); + if (!jl_bkind_is_some_import(kind)) + break; + if (kind != BINDING_KIND_IMPLICIT) + saw_explicit = 1; + bnd = (jl_binding_t*)bpart->restriction; + bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); + } + Value *bp = NULL; + if (bpart) { + enum jl_partition_kind kind = jl_binding_kind(bpart); + if (jl_bkind_is_some_constant(kind) && kind != BINDING_KIND_BACKDATED_CONST) { + if (possibly_deprecated) { + bp = julia_binding_gv(ctx, bnd); + ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); + } + jl_value_t *constval = bpart->restriction; + if (!constval) { + undef_var_error_ifnot(ctx, ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 0), name, (jl_value_t*)mod); + return jl_cgval_t(); } + return mark_julia_const(ctx, constval); } } if (!bpart || jl_binding_kind(bpart) != BINDING_KIND_GLOBAL) { return emit_globalref_runtime(ctx, bnd, mod, name); } - Value *bp = julia_binding_gv(ctx, bnd); - if (bnd->deprecated) { - cg_bdw(ctx, name, bnd); + bp = julia_binding_gv(ctx, bnd); + if (possibly_deprecated) { + ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); } jl_value_t *ty = bpart->restriction; - bp = julia_binding_pvalue(ctx, bp); + Value *bpval = julia_binding_pvalue(ctx, bp); if (ty == nullptr) ty = (jl_value_t*)jl_any_type; - return update_julia_type(ctx, emit_checked_var(ctx, bp, name, (jl_value_t*)mod, false, ctx.tbaa().tbaa_binding), ty); + return update_julia_type(ctx, emit_checked_var(ctx, bpval, name, (jl_value_t*)mod, false, ctx.tbaa().tbaa_binding), ty); } static jl_cgval_t emit_globalop(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *sym, jl_cgval_t rval, const jl_cgval_t &cmp, @@ -3308,6 +3304,7 @@ static jl_cgval_t emit_globalop(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *s Value *bp = julia_binding_gv(ctx, bnd); if (bpart) { if (jl_binding_kind(bpart) == BINDING_KIND_GLOBAL) { + int possibly_deprecated = bpart->kind & BINDING_FLAG_DEPWARN; jl_value_t *ty = bpart->restriction; if (ty != nullptr) { const std::string fname = issetglobal ? "setglobal!" : isreplaceglobal ? "replaceglobal!" : isswapglobal ? "swapglobal!" : ismodifyglobal ? "modifyglobal!" : "setglobalonce!"; @@ -3320,6 +3317,9 @@ static jl_cgval_t emit_globalop(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *s } bool isboxed = true; bool maybe_null = jl_atomic_load_relaxed(&bnd->value) == NULL; + if (possibly_deprecated) { + ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); + } return typed_store(ctx, julia_binding_pvalue(ctx, bp), rval, cmp, ty, @@ -9885,7 +9885,6 @@ static void init_jit_functions(void) add_named_global(memcmp_func, &memcmp); add_named_global(jltypeerror_func, &jl_type_error); add_named_global(jlcheckassign_func, &jl_checked_assignment); - add_named_global(jlgetbindingorerror_func, &jl_get_binding_or_error); add_named_global(jlcheckbpwritable_func, &jl_check_binding_currently_writable); add_named_global(jlboundp_func, &jl_boundp); for (auto it : builtin_func_map()) diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 27d2e949bb569..06c563dcc0fa5 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -193,7 +193,6 @@ XX(jl_get_backtrace) \ XX(jl_get_binding) \ XX(jl_get_binding_for_method_def) \ - XX(jl_get_binding_or_error) \ XX(jl_get_binding_wr) \ XX(jl_check_binding_currently_writable) \ XX(jl_get_cpu_name) \ diff --git a/src/julia.h b/src/julia.h index 11049da643372..93b3ae12d8eca 100644 --- a/src/julia.h +++ b/src/julia.h @@ -707,8 +707,21 @@ enum jl_partition_kind { BINDING_KIND_IMPLICIT_RECOMPUTE = 0xb }; -// These are flags that get anded into the above +static const uint8_t BINDING_KIND_MASK = 0x0f; +static const uint8_t BINDING_FLAG_MASK = 0xf0; + +//// These are flags that get anded into the above +// +// _EXPORTED: This binding partition is exported. In the world ranges covered by this partitions, +// other modules that `using` this module, may implicit import this binding. static const uint8_t BINDING_FLAG_EXPORTED = 0x10; +// _DEPRECATED: This binding partition is deprecated. It is considered weak for the purposes of +// implicit import resolution. +static const uint8_t BINDING_FLAG_DEPRECATED = 0x20; +// _DEPWARN: This binding partition will print a deprecation warning on access. Note that _DEPWARN +// implies _DEPRECATED. However, the reverse is not true. Such bindings are usually used for functions, +// where calling the function itself will provide a (better) deprecation warning/error. +static const uint8_t BINDING_FLAG_DEPWARN = 0x40; typedef struct __attribute__((aligned(8))) _jl_binding_partition_t { JL_DATA_TYPE @@ -2054,7 +2067,6 @@ JL_DLLEXPORT jl_value_t *jl_get_module_binding_or_nothing(jl_module_t *m, jl_sym // get binding for reading JL_DLLEXPORT jl_binding_t *jl_get_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); -JL_DLLEXPORT jl_binding_t *jl_get_binding_or_error(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT jl_value_t *jl_module_globalref(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT jl_value_t *jl_get_binding_type(jl_module_t *m, jl_sym_t *var); // get binding for assignment diff --git a/src/julia_internal.h b/src/julia_internal.h index 782e0bd0b1a96..6988eedc65e9b 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -913,7 +913,7 @@ JL_DLLEXPORT jl_value_t *jl_nth_slot_type(jl_value_t *sig JL_PROPAGATES_ROOT, si void jl_compute_field_offsets(jl_datatype_t *st); void jl_module_run_initializer(jl_module_t *m); JL_DLLEXPORT jl_binding_t *jl_get_module_binding(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var, int alloc); -JL_DLLEXPORT void jl_binding_deprecation_warning(jl_module_t *m, jl_sym_t *sym, jl_binding_t *b); +JL_DLLEXPORT void jl_binding_deprecation_warning(jl_binding_t *b); JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked(jl_binding_t *b JL_PROPAGATES_ROOT, jl_binding_partition_t *old_bpart, jl_value_t *restriction_val, enum jl_partition_kind kind, size_t new_world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b JL_PROPAGATES_ROOT, @@ -964,7 +964,8 @@ EXTERN_INLINE_DECLARE uint8_t jl_bpart_get_kind(jl_binding_partition_t *bpart) J } STATIC_INLINE void jl_walk_binding_inplace(jl_binding_t **bnd, jl_binding_partition_t **bpart JL_PROPAGATES_ROOT, size_t world) JL_NOTSAFEPOINT; -STATIC_INLINE void jl_walk_binding_inplace_all(jl_binding_t **bnd, jl_binding_partition_t **bpart JL_PROPAGATES_ROOT, size_t min_world, size_t max_world) JL_NOTSAFEPOINT; +STATIC_INLINE void jl_walk_binding_inplace_depwarn(jl_binding_t **bnd, jl_binding_partition_t **bpart, size_t world, int *depwarn) JL_NOTSAFEPOINT; +STATIC_INLINE void jl_walk_binding_inplace_all(jl_binding_t **bnd, jl_binding_partition_t **bpart JL_PROPAGATES_ROOT, int *depwarn, size_t min_world, size_t max_world) JL_NOTSAFEPOINT; #ifndef __clang_analyzer__ STATIC_INLINE void jl_walk_binding_inplace(jl_binding_t **bnd, jl_binding_partition_t **bpart, size_t world) JL_NOTSAFEPOINT @@ -977,13 +978,40 @@ STATIC_INLINE void jl_walk_binding_inplace(jl_binding_t **bnd, jl_binding_partit } } -STATIC_INLINE void jl_walk_binding_inplace_all(jl_binding_t **bnd, jl_binding_partition_t **bpart, size_t min_world, size_t max_world) JL_NOTSAFEPOINT +STATIC_INLINE void jl_walk_binding_inplace_depwarn(jl_binding_t **bnd, jl_binding_partition_t **bpart, size_t world, int *depwarn) JL_NOTSAFEPOINT { + int passed_explicit = 0; while (1) { - if (!(*bpart)) + enum jl_partition_kind kind = jl_binding_kind(*bpart); + if (!jl_bkind_is_some_import(kind)) { + if (!passed_explicit && depwarn) + *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; return; - if (!jl_bkind_is_some_import(jl_binding_kind(*bpart))) + } + if (!passed_explicit && depwarn) + *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; + if (kind != BINDING_KIND_IMPLICIT) + passed_explicit = 1; + *bnd = (jl_binding_t*)(*bpart)->restriction; + *bpart = jl_get_binding_partition(*bnd, world); + } +} + + +STATIC_INLINE void jl_walk_binding_inplace_all(jl_binding_t **bnd, jl_binding_partition_t **bpart, int *depwarn, size_t min_world, size_t max_world) JL_NOTSAFEPOINT +{ + int passed_explicit = 0; + while (*bpart) { + enum jl_partition_kind kind = jl_binding_kind(*bpart); + if (!jl_bkind_is_some_import(kind)) { + if (!passed_explicit && depwarn) + *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; return; + } + if (!passed_explicit && depwarn) + *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; + if (kind != BINDING_KIND_IMPLICIT) + passed_explicit = 1; *bnd = (jl_binding_t*)(*bpart)->restriction; *bpart = jl_get_binding_partition_all(*bnd, min_world, max_world); } diff --git a/src/module.c b/src/module.c index ee28c24798f95..0e03601187964 100644 --- a/src/module.c +++ b/src/module.c @@ -110,7 +110,7 @@ void jl_check_new_binding_implicit( continue; if (impb) { - if (tempb->deprecated) + if (tempbpart->kind & BINDING_FLAG_DEPRECATED) continue; if (jl_binding_kind(tempbpart) == BINDING_KIND_GUARD && jl_binding_kind(impbpart) != BINDING_KIND_GUARD) @@ -128,7 +128,7 @@ void jl_check_new_binding_implicit( guard_kind = BINDING_KIND_FAILED; break; } - else if (tempb->deprecated) { + else if (tempbpart->kind & BINDING_FLAG_DEPRECATED) { if (deprecated_impb) { if (!eq_bindings(tempbpart, deprecated_impb, world)) { guard_kind = BINDING_KIND_FAILED; @@ -315,7 +315,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( jl_symbol_name(mod->name), jl_symbol_name(var)); } if (bpart->min_world == new_world) { - bpart->kind = constant_kind | (bpart->kind & 0xf0); + bpart->kind = constant_kind | (bpart->kind & BINDING_FLAG_MASK); bpart->restriction = val; if (val) jl_gc_wb(bpart, val); @@ -497,7 +497,6 @@ static jl_binding_t *new_binding(jl_module_t *mod, jl_sym_t *name) b->globalref = NULL; b->backedges = NULL; b->publicp = 0; - b->deprecated = 0; b->did_print_backdate_admonition = 0; b->did_print_implicit_import_admonition = 0; JL_GC_PUSH1(&b); @@ -544,6 +543,9 @@ static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b, jl_module_t *m, jl_sym JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module_t *m, jl_sym_t *s) { jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + if (jl_options.depwarn && (bpart->kind & BINDING_FLAG_DEPWARN)) { + jl_binding_deprecation_warning(b); + } enum jl_partition_kind kind = jl_binding_kind(bpart); if (kind != BINDING_KIND_GLOBAL && kind != BINDING_KIND_DECLARED) { if (jl_bkind_is_some_guard(kind)) { @@ -620,6 +622,29 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_value(jl_binding_t *b) return jl_atomic_load_relaxed(&b->value); } +JL_DLLEXPORT jl_value_t *jl_get_binding_value_depwarn(jl_binding_t *b) +{ + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + if (jl_options.depwarn) { + int needs_depwarn = 0; + jl_walk_binding_inplace_depwarn(&b, &bpart, jl_current_task->world_age, &needs_depwarn); + if (needs_depwarn) + jl_binding_deprecation_warning(b); + } else { + jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + } + enum jl_partition_kind kind = jl_binding_kind(bpart); + if (jl_bkind_is_some_guard(kind)) + return NULL; + if (jl_bkind_is_some_constant(kind)) { + check_backdated_binding(b, kind); + return bpart->restriction; + } + assert(!jl_bkind_is_some_import(kind)); + return jl_atomic_load_relaxed(&b->value); +} + + JL_DLLEXPORT jl_value_t *jl_get_binding_value_seqcst(jl_binding_t *b) { jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); @@ -777,7 +802,7 @@ static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b, jl_module_t *m, jl_sym return m; } -static void jl_binding_dep_message(jl_module_t *m, jl_sym_t *name, jl_binding_t *b); +static void jl_binding_dep_message(jl_binding_t *b); // get type of binding m.var, without resolving the binding JL_DLLEXPORT jl_value_t *jl_get_binding_type(jl_module_t *m, jl_sym_t *var) @@ -805,17 +830,6 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding(jl_module_t *m, jl_sym_t *var) return jl_get_module_binding(m, var, 1); } -JL_DLLEXPORT jl_binding_t *jl_get_binding_or_error(jl_module_t *m, jl_sym_t *var) -{ - jl_binding_t *b = jl_get_binding(m, var); - if (b == NULL) - jl_undefined_var_error(var, (jl_value_t*)m); - // XXX: this only considers if the original is deprecated, not the binding in m - if (b->deprecated) - jl_binding_deprecation_warning(m, var, b); - return b; -} - JL_DLLEXPORT jl_value_t *jl_module_globalref(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_module_binding(m, var, 1); @@ -837,8 +851,10 @@ extern int jl_lineno; static char const dep_message_prefix[] = "_dep_message_"; -static void jl_binding_dep_message(jl_module_t *m, jl_sym_t *name, jl_binding_t *b) +static void jl_binding_dep_message(jl_binding_t *b) { + jl_module_t *m = b->globalref->mod; + jl_sym_t *name = b->globalref->name; size_t prefix_len = strlen(dep_message_prefix); size_t name_len = strlen(jl_symbol_name(name)); char *dep_binding_name = (char*)alloca(prefix_len+name_len+1); @@ -897,8 +913,8 @@ static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl { check_safe_import_from(from); jl_binding_t *b = jl_get_binding(from, s); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, ct->world_age); - if (b->deprecated) { + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + if (bpart->kind & BINDING_FLAG_DEPRECATED) { if (jl_get_binding_value(b) == jl_nothing) { // silently skip importing deprecated values assigned to nothing (to allow later mutation) return; @@ -914,7 +930,7 @@ static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl jl_symbol_name(to->name), asname == s ? "" : " as ", asname == s ? "" : jl_symbol_name(asname)); - jl_binding_dep_message(from, s, b); + jl_binding_dep_message(b); } } @@ -1209,19 +1225,13 @@ JL_DLLEXPORT jl_value_t *jl_get_globalref_value(jl_globalref_t *gr) jl_binding_t *b = gr->binding; if (!b) b = jl_get_module_binding(gr->mod, gr->name, 1); - // ignores b->deprecated - return b == NULL ? NULL : jl_get_binding_value(b); + return jl_get_binding_value_depwarn(b); } JL_DLLEXPORT jl_value_t *jl_get_global(jl_module_t *m, jl_sym_t *var) { - jl_binding_t *b = jl_get_binding(m, var); - if (b == NULL) - return NULL; - // XXX: this only considers if the original is deprecated, not the binding in m - if (b->deprecated) - jl_binding_deprecation_warning(m, var, b); - return jl_get_binding_value(b); + jl_binding_t *b = jl_get_module_binding(m, var, 1); + return jl_get_binding_value_depwarn(b); } JL_DLLEXPORT void jl_set_global(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var, jl_value_t *val JL_ROOTED_ARGUMENT) @@ -1237,7 +1247,7 @@ JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var jl_binding_partition_t *bpart = jl_get_binding_partition(bp, jl_current_task->world_age); bpart->min_world = 0; jl_atomic_store_release(&bpart->max_world, ~(size_t)0); - bpart->kind = BINDING_KIND_CONST | (bpart->kind & 0xf0); + bpart->kind = BINDING_KIND_CONST | (bpart->kind & BINDING_FLAG_MASK); bpart->restriction = val; jl_gc_wb(bpart, val); } @@ -1296,7 +1306,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked(jl_binding_t *b, jl_binding_partition_t *old_bpart, jl_value_t *restriction_val, enum jl_partition_kind kind, size_t new_world) { // Copy flags from old bpart - return jl_replace_binding_locked2(b, old_bpart, restriction_val, (size_t)kind | (size_t)(old_bpart->kind & 0xf0), + return jl_replace_binding_locked2(b, old_bpart, restriction_val, (size_t)kind | (size_t)(old_bpart->kind & BINDING_FLAG_MASK), new_world); } @@ -1318,10 +1328,10 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, jl_binding_partition_t *new_bpart = new_binding_partition(); JL_GC_PUSH1(&new_bpart); new_bpart->min_world = new_world; - if ((kind & 0x0f) == BINDING_KIND_IMPLICIT_RECOMPUTE) { + if ((kind & BINDING_KIND_MASK) == BINDING_KIND_IMPLICIT_RECOMPUTE) { assert(!restriction_val); jl_check_new_binding_implicit(new_bpart /* callee rooted */, b, NULL, new_world); - new_bpart->kind |= kind & 0xf0; + new_bpart->kind |= kind & BINDING_FLAG_MASK; } else { new_bpart->kind = kind; @@ -1406,44 +1416,95 @@ JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var) // set the deprecated flag for a binding: // 0=not deprecated, 1=renamed, 2=moved to another package +static const size_t DEPWARN_FLAGS = BINDING_FLAG_DEPRECATED | BINDING_FLAG_DEPWARN; JL_DLLEXPORT void jl_deprecate_binding(jl_module_t *m, jl_sym_t *var, int flag) { - // XXX: this deprecates the original value, which might be imported from elsewhere jl_binding_t *b = jl_get_binding(m, var); - if (b) b->deprecated = flag; + size_t new_flags = flag == 1 ? BINDING_FLAG_DEPRECATED | BINDING_FLAG_DEPWARN : + flag == 2 ? BINDING_FLAG_DEPRECATED : + 0; + JL_LOCK(&world_counter_lock); + size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; + jl_binding_partition_t *old_bpart = jl_get_binding_partition(b, jl_current_task->world_age); + if ((old_bpart->kind & DEPWARN_FLAGS) == new_flags) { + JL_UNLOCK(&world_counter_lock); + return; + } + jl_replace_binding_locked2(b, old_bpart, old_bpart->restriction, + (old_bpart->kind & ~DEPWARN_FLAGS) | new_flags, new_world); + jl_atomic_store_release(&jl_world_counter, new_world); + JL_UNLOCK(&world_counter_lock); +} + +static int should_depwarn(jl_binding_t *b, uint8_t flag) +{ + // We consider bindings deprecated, if: + // + // 1. The binding itself is deprecated, or + // 2. We implicitly import any deprecated binding. + // + // However, we do not consider the binding deprecated if the import was an explicit + // (`using` or `import`). The logic here is that the thing that needs to be adjusted + // is not the use itself, but rather the `using` or `import` (which already prints + // an appropriate warning). + for (;;) { + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + if (bpart->kind & BINDING_FLAG_DEPRECATED) + return 1; + if ((bpart->kind & BINDING_KIND_MASK) != BINDING_KIND_IMPLICIT) + break; + b = (jl_binding_t*)bpart->restriction; + } + return 0; +} + +JL_DLLEXPORT void jl_binding_deprecation_check(jl_binding_t *b) +{ + if (jl_options.depwarn && should_depwarn(b, BINDING_FLAG_DEPWARN)) + jl_binding_deprecation_warning(b); } JL_DLLEXPORT int jl_is_binding_deprecated(jl_module_t *m, jl_sym_t *var) { - // XXX: this only considers if the original is deprecated, not this precise binding - jl_binding_t *b = jl_get_binding(m, var); - return b && b->deprecated; + jl_binding_t *b = jl_get_module_binding(m, var, 0); + if (!b) + return 0; + return should_depwarn(b, BINDING_FLAG_DEPRECATED); } -void jl_binding_deprecation_warning(jl_module_t *m, jl_sym_t *s, jl_binding_t *b) +void jl_binding_deprecation_warning(jl_binding_t *b) { - // Only print a warning for deprecated == 1 (renamed). - // For deprecated == 2 (moved to a package) the binding is to a function - // that throws an error, so we don't want to print a warning too. - if (b->deprecated == 1 && jl_options.depwarn) { - if (jl_options.depwarn != JL_OPTIONS_DEPWARN_ERROR) - jl_printf(JL_STDERR, "WARNING: "); - jl_printf(JL_STDERR, "%s.%s is deprecated", - jl_symbol_name(m->name), jl_symbol_name(s)); - jl_binding_dep_message(m, s, b); + if (jl_options.depwarn != JL_OPTIONS_DEPWARN_ERROR) + jl_printf(JL_STDERR, "WARNING: "); + jl_printf(JL_STDERR, "Use of "); - if (jl_options.depwarn != JL_OPTIONS_DEPWARN_ERROR) { - if (jl_lineno != 0) { - jl_printf(JL_STDERR, " likely near %s:%d\n", jl_filename, jl_lineno); - } + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + int first = 0; + while (!(bpart->kind & BINDING_FLAG_DEPWARN)) { + if (first) { + jl_printf(JL_STDERR, "binding implicitly imported via "); + first = 0; } + jl_printf(JL_STDERR, "%s.%s -> ", jl_symbol_name(b->globalref->mod->name), jl_symbol_name(b->globalref->name)); + assert(jl_binding_kind(bpart) == BINDING_KIND_IMPLICIT); + b = (jl_binding_t*)bpart->restriction; + bpart = jl_get_binding_partition(b, jl_current_task->world_age); + } + jl_printf(JL_STDERR, "%s.%s is deprecated", + jl_symbol_name(b->globalref->mod->name), jl_symbol_name(b->globalref->name)); + jl_binding_dep_message(b); - if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) { - jl_errorf("use of deprecated variable: %s.%s", - jl_symbol_name(m->name), - jl_symbol_name(s)); + if (jl_options.depwarn != JL_OPTIONS_DEPWARN_ERROR) { + if (jl_lineno != 0) { + jl_printf(JL_STDERR, " likely near %s:%d\n", jl_filename, jl_lineno); } } + + if (jl_options.depwarn == JL_OPTIONS_DEPWARN_ERROR) { + jl_errorf("use of deprecated variable: %s.%s", + jl_symbol_name(b->globalref->mod->name), + jl_symbol_name(b->globalref->name)); + } } // For a generally writable binding (checked using jl_check_binding_currently_writable in this world age), check whether @@ -1549,7 +1610,7 @@ void append_module_names(jl_array_t* a, jl_module_t *m, int all, int imported, i (imported && (kind == BINDING_KIND_CONST_IMPORT || kind == BINDING_KIND_IMPORTED)) || (usings && kind == BINDING_KIND_EXPLICIT) || ((kind == BINDING_KIND_GLOBAL || kind == BINDING_KIND_CONST || kind == BINDING_KIND_DECLARED) && (all || main_public))) && - (all || (!b->deprecated && !hidden))) + (all || (!(bpart->kind & BINDING_FLAG_DEPRECATED) && !hidden))) _append_symbol_to_bindings_array(a, asname); } } @@ -1562,7 +1623,7 @@ void append_exported_names(jl_array_t* a, jl_module_t *m, int all) if ((void*)b == jl_nothing) break; jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if ((bpart->kind & BINDING_FLAG_EXPORTED) && (all || !b->deprecated)) + if ((bpart->kind & BINDING_FLAG_EXPORTED) && (all || !(bpart->kind & BINDING_FLAG_DEPRECATED))) _append_symbol_to_bindings_array(a, b->globalref->name); } } diff --git a/src/staticdata.c b/src/staticdata.c index 7dc23ed4c9449..fa6b709160804 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3553,10 +3553,10 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t if (jl_atomic_load_relaxed(&bpart->max_world) != ~(size_t)0) return 1; size_t raw_kind = bpart->kind; - enum jl_partition_kind kind = (enum jl_partition_kind)(raw_kind & 0x0f); + enum jl_partition_kind kind = (enum jl_partition_kind)(raw_kind & BINDING_KIND_MASK); if (!unchanged_implicit && jl_bkind_is_some_implicit(kind)) { jl_check_new_binding_implicit(bpart, b, NULL, jl_atomic_load_relaxed(&jl_world_counter)); - bpart->kind |= (raw_kind & 0xf0); + bpart->kind |= (raw_kind & BINDING_FLAG_MASK); if (bpart->min_world > jl_require_world) goto invalidated; } diff --git a/src/toplevel.c b/src/toplevel.c index 575f0ac2adbe6..a25a3794144ae 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -319,7 +319,7 @@ void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type, in } check_safe_newbinding(gm, gs); if (bpart->min_world == new_world) { - bpart->kind = new_kind | (bpart->kind & 0xf0); + bpart->kind = new_kind | (bpart->kind & BINDING_FLAG_MASK); bpart->restriction = global_type; if (global_type) jl_gc_wb(bpart, global_type); diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index fb44600886429..0556a243cbf37 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -753,7 +753,7 @@ let exename = `$(Base.julia_cmd()) --startup-file=no --color=no` @test errors_not_signals(`$exename -E "$code" --depwarn=error`) @test readchomperrors(`$exename -E "$code" --depwarn=yes`) == - (true, "true", "WARNING: Foo.Deprecated is deprecated, use NotDeprecated instead.\n likely near none:8") + (true, "true", "WARNING: Use of Foo.Deprecated is deprecated, use NotDeprecated instead.\n likely near none:8") @test readchomperrors(`$exename -E "$code" --depwarn=no`) == (true, "true", "") From 855807c802b83a501748034e8d30eca9add97366 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sun, 2 Mar 2025 15:50:03 -0500 Subject: [PATCH 25/74] bpart: Allow inference/codegen to merge multiple partitions (#57602) When inference and codegen queries the partitions of a binding, they often only care about a subset of the information in the partition. At the moment, we always truncate world ranges to the most precise partition that contains the relevant query. However, we can instead expand the world range to cover all partitions that are equivalent for the given query. To give a concrete example, both inference and codegen never care about the exported flag in a binding partition, so we should not unnecessarily truncate a world range just because an export was introduced. Further, this commit lays the ground work to stop invalidating code for these same kinds of transitions, although the actual logic to do that will come in a separate PR. (cherry picked from commit a5157c0fc5631195aef5872fb2f7a4b09d1818ea) --- Compiler/src/abstractinterpretation.jl | 219 ++++++++++++++++--------- Compiler/src/cicache.jl | 8 + Compiler/src/ssair/verify.jl | 19 +-- Compiler/src/stmtinfo.jl | 2 - Compiler/test/effects.jl | 11 ++ base/Base.jl | 1 - base/Base_compiler.jl | 1 + base/runtime_internals.jl | 4 + src/codegen.cpp | 119 ++++++-------- src/dlload.c | 4 +- src/julia.h | 1 + src/julia_internal.h | 35 ++++ src/module.c | 76 ++++++++- stdlib/REPL/src/REPLCompletions.jl | 11 +- test/rebinding.jl | 21 +++ 15 files changed, 351 insertions(+), 181 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index b5882deb081df..d733f79f949bf 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2393,8 +2393,8 @@ function abstract_eval_getglobal(interp::AbstractInterpreter, sv::AbsIntState, s M, s = M.val, s.val if M isa Module && s isa Symbol gr = GlobalRef(M, s) - (ret, bpart) = abstract_eval_globalref(interp, gr, saw_latestworld, sv) - return CallMeta(ret, bpart === nothing ? NoCallInfo() : GlobalAccessInfo(convert(Core.Binding, gr), bpart)) + ret = abstract_eval_globalref(interp, gr, saw_latestworld, sv) + return CallMeta(ret, GlobalAccessInfo(convert(Core.Binding, gr))) end return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) elseif !hasintersect(widenconst(M), Module) || !hasintersect(widenconst(s), Symbol) @@ -2437,18 +2437,23 @@ end if !isa(M, Module) || !isa(s, Symbol) return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) end - partition = abstract_eval_binding_partition!(interp, GlobalRef(M, s), sv) - - if is_some_guard(binding_kind(partition)) - # We do not currently assume an invalidation for guard -> defined transitions - # rt = Const(nothing) - rt = Type - elseif is_some_const_binding(binding_kind(partition)) - rt = Const(Any) - else - rt = Const(partition_restriction(partition)) + gr = GlobalRef(M, s) + (valid_worlds, rt) = scan_leaf_partitions(interp, gr, sv.world) do interp, _, partition + local rt + kind = binding_kind(partition) + if is_some_guard(kind) || kind == BINDING_KIND_DECLARED + # We do not currently assume an invalidation for guard -> defined transitions + # rt = Const(nothing) + rt = Type + elseif is_some_const_binding(kind) + rt = Const(Any) + else + rt = Const(partition_restriction(partition)) + end + rt end - return CallMeta(rt, Union{}, EFFECTS_TOTAL, NoCallInfo()) + update_valid_age!(sv, valid_worlds) + return CallMeta(rt, Union{}, EFFECTS_TOTAL, GlobalAccessInfo(convert(Core.Binding, gr))) elseif !hasintersect(widenconst(M), Module) || !hasintersect(widenconst(s), Symbol) return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) elseif M ⊑ Module && s ⊑ Symbol @@ -2473,8 +2478,8 @@ function abstract_eval_setglobal!(interp::AbstractInterpreter, sv::AbsIntState, M, s = M.val, s.val if M isa Module && s isa Symbol gr = GlobalRef(M, s) - (rt, exct), partition = global_assignment_rt_exct(interp, sv, saw_latestworld, gr, v) - return CallMeta(rt, exct, Effects(setglobal!_effects, nothrow=exct===Bottom), GlobalAccessInfo(convert(Core.Binding, gr), partition)) + (rt, exct) = global_assignment_rt_exct(interp, sv, saw_latestworld, gr, v) + return CallMeta(rt, exct, Effects(setglobal!_effects, nothrow=exct===Bottom), GlobalAccessInfo(convert(Core.Binding, gr))) end return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) end @@ -2554,6 +2559,7 @@ function abstract_eval_setglobalonce!(interp::AbstractInterpreter, sv::AbsIntSta end end + function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, argtypes::Vector{Any}) if length(argtypes) in (5, 6, 7) (M, s, x, v) = argtypes[2], argtypes[3], argtypes[4], argtypes[5] @@ -2563,14 +2569,19 @@ function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntSta M isa Module || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) s isa Symbol || return CallMeta(Union{}, TypeError, EFFECTS_THROWS, NoCallInfo()) gr = GlobalRef(M, s) - partition = abstract_eval_binding_partition!(interp, gr, sv) - rte = abstract_eval_partition_load(interp, partition) - if binding_kind(partition) == BINDING_KIND_GLOBAL - T = partition_restriction(partition) + (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp, _, partition + partition_T = nothing + partition_rte = abstract_eval_partition_load(interp, partition) + if binding_kind(partition) == BINDING_KIND_GLOBAL + partition_T = partition_restriction(partition) + end + partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v)[2]} + partition_rte = RTEffects(partition_rte.rt, partition_exct, partition_rte.effects) + Pair{RTEffects, Any}(partition_rte, partition_T) end - exct = Union{rte.exct, global_assignment_binding_rt_exct(interp, partition, v)[2]} - effects = merge_effects(rte.effects, Effects(setglobal!_effects, nothrow=exct===Bottom)) - sg = CallMeta(Any, exct, effects, GlobalAccessInfo(convert(Core.Binding, gr), partition)) + update_valid_age!(sv, valid_worlds) + effects = merge_effects(rte.effects, Effects(setglobal!_effects, nothrow=rte.exct===Bottom)) + sg = CallMeta(Any, rte.exct, effects, GlobalAccessInfo(convert(Core.Binding, gr))) else sg = abstract_eval_setglobal!(interp, sv, saw_latestworld, M, s, v) end @@ -2941,7 +2952,7 @@ function abstract_eval_special_value(interp::AbstractInterpreter, @nospecialize( end elseif isa(e, GlobalRef) # No need for an edge since an explicit GlobalRef will be picked up by the source scan - return abstract_eval_globalref(interp, e, sstate.saw_latestworld, sv)[1] + return abstract_eval_globalref(interp, e, sstate.saw_latestworld, sv) end if isa(e, QuoteNode) e = e.value @@ -3226,25 +3237,29 @@ function abstract_eval_isdefinedglobal(interp::AbstractInterpreter, mod::Module, effects = EFFECTS_TOTAL gr = GlobalRef(mod, sym) - partition = lookup_binding_partition!(interp, gr, sv) - if allow_import !== true && is_some_imported(binding_kind(partition)) - if allow_import === false - rt = Const(false) - else - effects = Effects(generic_isdefinedglobal_effects, nothrow=true) + if allow_import !== true + gr = GlobalRef(mod, sym) + partition = lookup_binding_partition!(interp, gr, sv) + if allow_import !== true && is_some_imported(binding_kind(partition)) + if allow_import === false + rt = Const(false) + else + effects = Effects(generic_isdefinedglobal_effects, nothrow=true) + end + @goto done end + end + + (valid_worlds, rte) = abstract_load_all_consistent_leaf_partitions(interp, gr, sv.world) + if rte.exct == Union{} + rt = Const(true) + elseif rte.rt === Union{} && rte.exct === UndefVarError + rt = Const(false) else - partition = walk_binding_partition!(interp, partition, sv) - rte = abstract_eval_partition_load(interp, partition) - if rte.exct == Union{} - rt = Const(true) - elseif rte.rt === Union{} && rte.exct === UndefVarError - rt = Const(false) - else - effects = Effects(generic_isdefinedglobal_effects, nothrow=true) - end + effects = Effects(generic_isdefinedglobal_effects, nothrow=true) end - return CallMeta(RTEffects(rt, Union{}, effects), GlobalAccessInfo(convert(Core.Binding, gr), partition)) +@label done + return CallMeta(RTEffects(rt, Union{}, effects), GlobalAccessInfo(convert(Core.Binding, gr))) end function abstract_eval_isdefinedglobal(interp::AbstractInterpreter, @nospecialize(M), @nospecialize(s), @nospecialize(allow_import_arg), @nospecialize(order_arg), saw_latestworld::Bool, sv::AbsIntState) @@ -3461,50 +3476,47 @@ world_range(compact::IncrementalCompact) = world_range(compact.ir) function abstract_eval_globalref_type(g::GlobalRef, src::Union{CodeInfo, IRCode, IncrementalCompact}) worlds = world_range(src) partition = lookup_binding_partition(min_world(worlds), g) - partition.max_world < max_world(worlds) && return Any - while is_some_imported(binding_kind(partition)) - imported_binding = partition_restriction(partition)::Core.Binding - partition = lookup_binding_partition(min_world(worlds), imported_binding) - partition.max_world < max_world(worlds) && return Any - end - kind = binding_kind(partition) - if is_some_guard(kind) - # return Union{} + + (valid_worlds, rte) = abstract_load_all_consistent_leaf_partitions(nothing, g, WorldWithRange(min_world(worlds), worlds)) + if min_world(valid_worlds) > min_world(worlds) || max_world(valid_worlds) < max_world(worlds) return Any end - if is_some_const_binding(kind) - return Const(partition_restriction(partition)) - end - return kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition) + + return rte.rt end -function lookup_binding_partition!(interp::AbstractInterpreter, g::GlobalRef, sv::AbsIntState) +function lookup_binding_partition!(interp::AbstractInterpreter, g::Union{GlobalRef, Core.Binding}, sv::AbsIntState) partition = lookup_binding_partition(get_inference_world(interp), g) update_valid_age!(sv, WorldRange(partition.min_world, partition.max_world)) partition end -function walk_binding_partition!(interp::AbstractInterpreter, partition::Core.BindingPartition, sv::AbsIntState) +function walk_binding_partition(imported_binding::Core.Binding, partition::Core.BindingPartition, world::UInt) + valid_worlds = WorldRange(partition.min_world, partition.max_world) while is_some_imported(binding_kind(partition)) imported_binding = partition_restriction(partition)::Core.Binding - partition = lookup_binding_partition(get_inference_world(interp), imported_binding) - update_valid_age!(sv, WorldRange(partition.min_world, partition.max_world)) + partition = lookup_binding_partition(world, imported_binding) + valid_worlds = intersect(valid_worlds, WorldRange(partition.min_world, partition.max_world)) end - return partition + return Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}(valid_worlds, imported_binding=>partition) end function abstract_eval_binding_partition!(interp::AbstractInterpreter, g::GlobalRef, sv::AbsIntState) - partition = lookup_binding_partition!(interp, g, sv) - partition = walk_binding_partition!(interp, partition, sv) + b = convert(Core.Binding, g) + partition = lookup_binding_partition!(interp, b, sv) + valid_worlds, (_, partition) = walk_binding_partition(b, partition, get_inference_world(interp)) + update_valid_age!(sv, valid_worlds) return partition end -function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Core.BindingPartition) +abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, ::Core.Binding, partition::Core.BindingPartition) = + abstract_eval_partition_load(interp, partition) +function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, partition::Core.BindingPartition) kind = binding_kind(partition) isdepwarn = (partition.kind & BINDING_FLAG_DEPWARN) != 0 local_getglobal_effects = Effects(generic_getglobal_effects, effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE) if is_some_guard(kind) || kind == BINDING_KIND_UNDEF_CONST - if InferenceParams(interp).assume_bindings_static + if interp !== nothing && InferenceParams(interp).assume_bindings_static return RTEffects(Union{}, UndefVarError, EFFECTS_THROWS) else # We do not currently assume an invalidation for guard -> defined transitions @@ -3526,6 +3538,10 @@ function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Co end if kind == BINDING_KIND_DECLARED + # Could be replaced by a backdated const which has an effect, so we can't assume it won't. + # Besides, we would prefer not to merge the world range for this into the world range for + # _GLOBAL, because that would pessimize codegen. + local_getglobal_effects = Effects(local_getglobal_effects, effect_free=ALWAYS_FALSE) rt = Any else rt = partition_restriction(partition) @@ -3533,39 +3549,88 @@ function abstract_eval_partition_load(interp::AbstractInterpreter, partition::Co return RTEffects(rt, UndefVarError, local_getglobal_effects) end -function abstract_eval_globalref(interp::AbstractInterpreter, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) - if saw_latestworld - return Pair{RTEffects, Union{Nothing, Core.BindingPartition}}(RTEffects(Any, Any, generic_getglobal_effects), nothing) +function scan_specified_partitions(query::Function, walk_binding_partition::Function, interp, g::GlobalRef, wwr::WorldWithRange) + local total_validity, rte, binding_partition + binding = convert(Core.Binding, g) + lookup_world = max_world(wwr.valid_worlds) + while true + # Partitions are ordered newest-to-oldest so start at the top + binding_partition = @isdefined(binding_partition) ? + lookup_binding_partition(lookup_world, binding, binding_partition) : + lookup_binding_partition(lookup_world, binding) + while lookup_world >= binding_partition.min_world && (!@isdefined(total_validity) || min_world(total_validity) > min_world(wwr.valid_worlds)) + partition_validity, (leaf_binding, leaf_partition) = walk_binding_partition(binding, binding_partition, lookup_world) + @assert lookup_world in partition_validity + this_rte = query(interp, leaf_binding, leaf_partition) + if @isdefined(rte) + if this_rte === rte + total_validity = union(total_validity, partition_validity) + lookup_world = min_world(total_validity) - 1 + continue + end + if min_world(total_validity) <= wwr.this + @goto out + end + end + total_validity = partition_validity + lookup_world = min_world(total_validity) - 1 + rte = this_rte + end + min_world(total_validity) > min_world(wwr.valid_worlds) || break end - partition = abstract_eval_binding_partition!(interp, g, sv) - ret = abstract_eval_partition_load(interp, partition) - if ret.rt !== Union{} && ret.exct === UndefVarError && InferenceParams(interp).assume_bindings_static - b = convert(Core.Binding, g) - if isdefined(b, :value) +@label out + return Pair{WorldRange, typeof(rte)}(total_validity, rte) +end + +scan_leaf_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) = + scan_specified_partitions(query, walk_binding_partition, interp, g, wwr) + +scan_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) = + scan_specified_partitions(query, + (b::Core.Binding, bpart::Core.BindingPartition, world::UInt)-> + Pair{WorldRange, Pair{Core.Binding, Core.BindingPartition}}(WorldRange(bpart.min_world, bpart.max_world), b=>bpart), + interp, g, wwr) + +abstract_load_all_consistent_leaf_partitions(interp, g::GlobalRef, wwr::WorldWithRange) = + scan_leaf_partitions(abstract_eval_partition_load, interp, g, wwr) + +function abstract_eval_globalref(interp, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) + if saw_latestworld + return RTEffects(Any, Any, generic_getglobal_effects) + end + (valid_worlds, (ret, binding_if_global)) = scan_leaf_partitions(interp, g, sv.world) do interp, binding, partition + # For inference purposes, we don't particularly care which global binding we end up loading, we only + # care about its type. However, we would still like to terminate the world range for the particular + # binding we end up reaching such that codegen can emit a simpler pointer load. + Pair{RTEffects, Union{Nothing, Core.Binding}}( + abstract_eval_partition_load(interp, partition), + binding_kind(partition) in (BINDING_KIND_GLOBAL, BINDING_KIND_DECLARED) ? binding : nothing) + end + update_valid_age!(sv, valid_worlds) + if ret.rt !== Union{} && ret.exct === UndefVarError && binding_if_global !== nothing && InferenceParams(interp).assume_bindings_static + if isdefined(binding_if_global, :value) ret = RTEffects(ret.rt, Union{}, Effects(generic_getglobal_effects, nothrow=true)) end # We do not assume in general that assigned global bindings remain assigned. # The existence of pkgimages allows them to revert in practice. end - return Pair{RTEffects, Union{Nothing, Core.BindingPartition}}(ret, partition) + return ret end function global_assignment_rt_exct(interp::AbstractInterpreter, sv::AbsIntState, saw_latestworld::Bool, g::GlobalRef, @nospecialize(newty)) if saw_latestworld - return Pair{Pair{Any,Any}, Union{Core.BindingPartition, Nothing}}( - Pair{Any,Any}(newty, Union{ErrorException, TypeError}), nothing) + return Pair{Any,Any}(newty, Union{ErrorException, TypeError}) end - partition = abstract_eval_binding_partition!(interp, g, sv) - return Pair{Pair{Any,Any}, Union{Core.BindingPartition, Nothing}}( - global_assignment_binding_rt_exct(interp, partition, newty), - partition) + (valid_worlds, ret) = scan_partitions((interp, _, partition)->global_assignment_binding_rt_exct(interp, partition, newty), interp, g, sv.world) + update_valid_age!(sv, valid_worlds) + return ret end function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partition::Core.BindingPartition, @nospecialize(newty)) kind = binding_kind(partition) if is_some_guard(kind) return Pair{Any,Any}(newty, ErrorException) - elseif is_some_const_binding(kind) + elseif is_some_const_binding(kind) || is_some_imported(kind) return Pair{Any,Any}(Bottom, ErrorException) end ty = kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition) diff --git a/Compiler/src/cicache.jl b/Compiler/src/cicache.jl index 2893be2787b29..9c528bc0ae822 100644 --- a/Compiler/src/cicache.jl +++ b/Compiler/src/cicache.jl @@ -40,6 +40,14 @@ function intersect(a::WorldRange, b::WorldRange) return ret end +function union(a::WorldRange, b::WorldRange) + if b.min_world < a.min_world + (b, a) = (a, b) + end + @assert a.max_world >= b.min_world - 1 + return WorldRange(a.min_world, b.max_world) +end + """ struct WorldView diff --git a/Compiler/src/ssair/verify.jl b/Compiler/src/ssair/verify.jl index 6b9863a87ec05..2b8f89173911a 100644 --- a/Compiler/src/ssair/verify.jl +++ b/Compiler/src/ssair/verify.jl @@ -62,17 +62,14 @@ function check_op(ir::IRCode, domtree::DomTree, @nospecialize(op), use_bb::Int, raise_error() end elseif isa(op, GlobalRef) - bpart = lookup_binding_partition(min_world(ir.valid_worlds), op) - while is_some_imported(binding_kind(bpart)) && max_world(ir.valid_worlds) <= bpart.max_world - imported_binding = partition_restriction(bpart)::Core.Binding - bpart = lookup_binding_partition(min_world(ir.valid_worlds), imported_binding) - end - if (!is_defined_const_binding(binding_kind(bpart)) || (bpart.max_world < max_world(ir.valid_worlds))) && - (op.mod !== Core) && (op.mod !== Base) - # Core and Base are excluded because the frontend uses them for intrinsics, etc. - # TODO: Decide which way to go with these. - @verify_error "Unbound or partitioned GlobalRef not allowed in value position" - raise_error() + if op.mod !== Core && op.mod !== Base + (valid_worlds, alldef) = scan_leaf_partitions(nothing, op, WorldWithRange(min_world(ir.valid_worlds), ir.valid_worlds)) do _, _, bpart + is_defined_const_binding(binding_kind(bpart)) + end + if !alldef || max_world(valid_worlds) < max_world(ir.valid_worlds) || min_world(valid_worlds) > min_world(ir.valid_worlds) + @verify_error "Unbound or partitioned GlobalRef not allowed in value position" + raise_error() + end end elseif isa(op, Expr) # Only Expr(:boundscheck) is allowed in value position diff --git a/Compiler/src/stmtinfo.jl b/Compiler/src/stmtinfo.jl index 58c7c7b3fea11..6a85bc6605d3f 100644 --- a/Compiler/src/stmtinfo.jl +++ b/Compiler/src/stmtinfo.jl @@ -490,9 +490,7 @@ perform such accesses. """ struct GlobalAccessInfo <: CallInfo b::Core.Binding - bpart::Core.BindingPartition end -GlobalAccessInfo(::Core.Binding, ::Nothing) = NoCallInfo() function add_edges_impl(edges::Vector{Any}, info::GlobalAccessInfo) push!(edges, info.b) end diff --git a/Compiler/test/effects.jl b/Compiler/test/effects.jl index 082d954b4a9bc..d9ac97f36e181 100644 --- a/Compiler/test/effects.jl +++ b/Compiler/test/effects.jl @@ -400,6 +400,17 @@ let effects = Base.infer_effects(setglobal!_nothrow_undefinedyet2) end @test_throws TypeError setglobal!_nothrow_undefinedyet2() +module ExportMutableGlobal + global mutable_global_for_setglobal_test::Int = 0 + export mutable_global_for_setglobal_test +end +using .ExportMutableGlobal: mutable_global_for_setglobal_test +f_assign_imported() = global mutable_global_for_setglobal_test = 42 +let effects = Base.infer_effects(f_assign_imported) + @test !Compiler.is_nothrow(effects) +end +@test_throws ErrorException f_assign_imported() + # Nothrow for setfield! mutable struct SetfieldNothrow x::Int diff --git a/base/Base.jl b/base/Base.jl index acdb54b93077b..7b1672bf9602c 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -21,7 +21,6 @@ include(strcat(BUILDROOT, "version_git.jl")) # include($BUILDROOT/base/version_g # Initialize DL_LOAD_PATH as early as possible. We are defining things here in # a slightly more verbose fashion than usual, because we're running so early. -const DL_LOAD_PATH = String[] let os = ccall(:jl_get_UNAME, Any, ()) if os === :Darwin || os === :Apple if Base.DARWIN_FRAMEWORK diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index fc2304f8fce46..2bd9a75a6dca6 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -334,6 +334,7 @@ end BUILDROOT::String = "" DATAROOT::String = "" +const DL_LOAD_PATH = String[] baremodule BuildSettings end diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index cce71bdecffb4..6850859a7ba98 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -225,6 +225,10 @@ function lookup_binding_partition(world::UInt, b::Core.Binding) ccall(:jl_get_binding_partition, Ref{Core.BindingPartition}, (Any, UInt), b, world) end +function lookup_binding_partition(world::UInt, b::Core.Binding, previous_partition::Core.BindingPartition) + ccall(:jl_get_binding_partition_with_hint, Ref{Core.BindingPartition}, (Any, Any, UInt), b, previous_partition, world) +end + function convert(::Type{Core.Binding}, gr::Core.GlobalRef) if isdefined(gr, :binding) return gr.binding diff --git a/src/codegen.cpp b/src/codegen.cpp index 58c59496ccc2f..d70dcc227847d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -2939,14 +2939,13 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) { if (jl_is_symbol(ex)) { jl_sym_t *sym = (jl_sym_t*)ex; - jl_binding_t *bnd = jl_get_module_binding(ctx.module, sym, 0); - jl_binding_partition_t *bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); + jl_binding_t *bnd = jl_get_module_binding(ctx.module, sym, 1); int possibly_deprecated = 0; - jl_walk_binding_inplace_all(&bnd, &bpart, &possibly_deprecated, ctx.min_world, ctx.max_world); - if (bpart && jl_bkind_is_some_constant(jl_binding_kind(bpart))) { + jl_value_t *cval = jl_get_binding_leaf_partitions_value_if_const(bnd, &possibly_deprecated, ctx.min_world, ctx.max_world); + if (cval) { if (possibly_deprecated) emit_depwarn_check(ctx, bnd); - return bpart->restriction; + return cval; } return NULL; } @@ -2968,13 +2967,9 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) jl_sym_t *s = NULL; if (jl_is_globalref(ex)) { s = jl_globalref_name(ex); - jl_binding_t *bnd = jl_get_module_binding(jl_globalref_mod(ex), s, 0); - jl_binding_partition_t *bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); + jl_binding_t *bnd = jl_get_module_binding(jl_globalref_mod(ex), s, 1); int possibly_deprecated = 0; - jl_walk_binding_inplace_all(&bnd, &bpart, &possibly_deprecated, ctx.min_world, ctx.max_world); - jl_value_t *v = NULL; - if (bpart && jl_bkind_is_some_constant(jl_binding_kind(bpart))) - v = bpart->restriction; + jl_value_t *v = jl_get_binding_leaf_partitions_value_if_const(bnd, &possibly_deprecated, ctx.min_world, ctx.max_world); if (v) { if (possibly_deprecated) emit_depwarn_check(ctx, bnd); @@ -2996,13 +2991,9 @@ static jl_value_t *static_eval(jl_codectx_t &ctx, jl_value_t *ex) // Assumes that the module is rooted somewhere. s = (jl_sym_t*)static_eval(ctx, jl_exprarg(e, 2)); if (s && jl_is_symbol(s)) { - jl_binding_t *bnd = jl_get_module_binding(m, s, 0); - jl_binding_partition_t *bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); + jl_binding_t *bnd = jl_get_module_binding(m, s, 1); int possibly_deprecated = 0; - jl_walk_binding_inplace_all(&bnd, &bpart, &possibly_deprecated, ctx.min_world, ctx.max_world); - jl_value_t *v = NULL; - if (bpart && jl_bkind_is_some_constant(jl_binding_kind(bpart))) - v = bpart->restriction; + jl_value_t *v = jl_get_binding_leaf_partitions_value_if_const(bnd, &possibly_deprecated, ctx.min_world, ctx.max_world); if (v) { if (possibly_deprecated) emit_depwarn_check(ctx, bnd); @@ -3246,48 +3237,32 @@ static jl_cgval_t emit_globalref_runtime(jl_codectx_t &ctx, jl_binding_t *bnd, j static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *name, AtomicOrdering order) { jl_binding_t *bnd = jl_get_module_binding(mod, name, 1); - assert(bnd); - jl_binding_partition_t *bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); - if (!bpart) { + struct restriction_kind_pair rkp = { NULL, NULL, BINDING_KIND_GUARD, 0 }; + if (!jl_get_binding_leaf_partitions_restriction_kind(bnd, &rkp, ctx.min_world, ctx.max_world)) { return emit_globalref_runtime(ctx, bnd, mod, name); } - int possibly_deprecated = 0; - int saw_explicit = 0; - while (bpart) { - if (!saw_explicit && (bpart->kind & BINDING_FLAG_DEPWARN)) - possibly_deprecated = 1; - enum jl_partition_kind kind = jl_binding_kind(bpart); - if (!jl_bkind_is_some_import(kind)) - break; - if (kind != BINDING_KIND_IMPLICIT) - saw_explicit = 1; - bnd = (jl_binding_t*)bpart->restriction; - bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); - } - Value *bp = NULL; - if (bpart) { - enum jl_partition_kind kind = jl_binding_kind(bpart); - if (jl_bkind_is_some_constant(kind) && kind != BINDING_KIND_BACKDATED_CONST) { - if (possibly_deprecated) { - bp = julia_binding_gv(ctx, bnd); - ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); - } - jl_value_t *constval = bpart->restriction; - if (!constval) { - undef_var_error_ifnot(ctx, ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 0), name, (jl_value_t*)mod); - return jl_cgval_t(); - } - return mark_julia_const(ctx, constval); + if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != BINDING_KIND_BACKDATED_CONST) { + if (rkp.maybe_depwarn) { + Value *bp = julia_binding_gv(ctx, bnd); + ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); + } + jl_value_t *constval = rkp.restriction; + if (!constval) { + undef_var_error_ifnot(ctx, ConstantInt::get(getInt1Ty(ctx.builder.getContext()), 0), name, (jl_value_t*)mod); + return jl_cgval_t(); } + return mark_julia_const(ctx, constval); } - if (!bpart || jl_binding_kind(bpart) != BINDING_KIND_GLOBAL) { + if (rkp.kind != BINDING_KIND_GLOBAL) { return emit_globalref_runtime(ctx, bnd, mod, name); } - bp = julia_binding_gv(ctx, bnd); - if (possibly_deprecated) { + Value *bp = julia_binding_gv(ctx, bnd); + if (rkp.maybe_depwarn) { ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); } - jl_value_t *ty = bpart->restriction; + if (bnd != rkp.binding_if_global) + bp = julia_binding_gv(ctx, rkp.binding_if_global); + jl_value_t *ty = rkp.restriction; Value *bpval = julia_binding_pvalue(ctx, bp); if (ty == nullptr) ty = (jl_value_t*)jl_any_type; @@ -3851,27 +3826,27 @@ static jl_cgval_t emit_isdefinedglobal(jl_codectx_t &ctx, jl_module_t *modu, jl_ { Value *isnull = NULL; jl_binding_t *bnd = allow_import ? jl_get_binding(modu, name) : jl_get_module_binding(modu, name, 0); - jl_binding_partition_t *bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); - enum jl_partition_kind kind = bpart ? jl_binding_kind(bpart) : BINDING_KIND_GUARD; - if (kind == BINDING_KIND_GLOBAL || jl_bkind_is_some_constant(kind)) { - if (jl_get_binding_value_if_const(bnd)) - return mark_julia_const(ctx, jl_true); - Value *bp = julia_binding_gv(ctx, bnd); - bp = julia_binding_pvalue(ctx, bp); - LoadInst *v = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*))); - jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_binding); - ai.decorateInst(v); - v->setOrdering(get_llvm_atomic_order(order)); - isnull = ctx.builder.CreateICmpNE(v, Constant::getNullValue(ctx.types().T_prjlvalue)); - } - else { - Value *v = ctx.builder.CreateCall(prepare_call(jlboundp_func), { - literal_pointer_val(ctx, (jl_value_t*)modu), - literal_pointer_val(ctx, (jl_value_t*)name), - ConstantInt::get(getInt32Ty(ctx.builder.getContext()), allow_import) - }); - isnull = ctx.builder.CreateICmpNE(v, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0)); - } + struct restriction_kind_pair rkp = { NULL, NULL, BINDING_KIND_GUARD, 0 }; + if (allow_import && jl_get_binding_leaf_partitions_restriction_kind(bnd, &rkp, ctx.min_world, ctx.max_world)) { + if (jl_bkind_is_some_constant(rkp.kind)) + return mark_julia_const(ctx, rkp.restriction); + if (rkp.kind == BINDING_KIND_GLOBAL) { + Value *bp = julia_binding_gv(ctx, rkp.binding_if_global); + bp = julia_binding_pvalue(ctx, bp); + LoadInst *v = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*))); + jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_binding); + ai.decorateInst(v); + v->setOrdering(get_llvm_atomic_order(order)); + isnull = ctx.builder.CreateICmpNE(v, Constant::getNullValue(ctx.types().T_prjlvalue)); + return mark_julia_type(ctx, isnull, false, jl_bool_type); + } + } + Value *v = ctx.builder.CreateCall(prepare_call(jlboundp_func), { + literal_pointer_val(ctx, (jl_value_t*)modu), + literal_pointer_val(ctx, (jl_value_t*)name), + ConstantInt::get(getInt32Ty(ctx.builder.getContext()), allow_import) + }); + isnull = ctx.builder.CreateICmpNE(v, ConstantInt::get(getInt32Ty(ctx.builder.getContext()), 0)); return mark_julia_type(ctx, isnull, false, jl_bool_type); } diff --git a/src/dlload.c b/src/dlload.c index 91980cc4ecbbf..7a25903d471aa 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -307,9 +307,9 @@ JL_DLLEXPORT void *jl_load_dynamic_library(const char *modname, unsigned flags, While these exist as OS concepts on Darwin, we want to use them on other platforms such as Windows, so we emulate them here. */ - if (!abspath && !is_atpath && jl_base_module != NULL) { + if (!abspath && !is_atpath && jl_base_module != NULL && jl_typeinf_world != 1) { jl_binding_t *b = jl_get_module_binding(jl_base_module, jl_symbol("DL_LOAD_PATH"), 0); - jl_array_t *DL_LOAD_PATH = (jl_array_t*)(b ? jl_get_binding_value(b) : NULL); + jl_array_t *DL_LOAD_PATH = (jl_array_t*)(b ? jl_get_binding_value_in_world(b, jl_typeinf_world) : NULL); if (DL_LOAD_PATH != NULL) { size_t j; for (j = 0; j < jl_array_nrows(DL_LOAD_PATH); j++) { diff --git a/src/julia.h b/src/julia.h index 93b3ae12d8eca..8ee57acbabb4d 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1920,6 +1920,7 @@ JL_DLLEXPORT jl_sym_t *jl_gensym(void); JL_DLLEXPORT jl_sym_t *jl_tagged_gensym(const char *str, size_t len); JL_DLLEXPORT jl_sym_t *jl_get_root_symbol(void); JL_DLLEXPORT jl_value_t *jl_get_binding_value(jl_binding_t *b JL_PROPAGATES_ROOT); +JL_DLLEXPORT jl_value_t *jl_get_binding_value_in_world(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world); JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT); JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; JL_DLLEXPORT jl_value_t *jl_get_binding_value_if_resolved_and_const(jl_binding_t *b JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT; diff --git a/src/julia_internal.h b/src/julia_internal.h index 6988eedc65e9b..315cc24c9f7a9 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -957,8 +957,18 @@ STATIC_INLINE int jl_bkind_is_defined_constant(enum jl_partition_kind kind) JL_N } JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world) JL_GLOBALLY_ROOTED; +JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition_with_hint(jl_binding_t *b JL_PROPAGATES_ROOT, jl_binding_partition_t *previous_part, size_t world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition_all(jl_binding_t *b JL_PROPAGATES_ROOT, size_t min_world, size_t max_world) JL_GLOBALLY_ROOTED; +struct restriction_kind_pair { + jl_binding_t *binding_if_global; + jl_value_t *restriction; + enum jl_partition_kind kind; + int maybe_depwarn; +}; +JL_DLLEXPORT int jl_get_binding_leaf_partitions_restriction_kind(jl_binding_t *b JL_PROPAGATES_ROOT, struct restriction_kind_pair *rkp, size_t min_world, size_t max_world) JL_GLOBALLY_ROOTED; +JL_DLLEXPORT jl_value_t *jl_get_binding_leaf_partitions_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT, int *maybe_depwarn, size_t min_world, size_t max_world); + EXTERN_INLINE_DECLARE uint8_t jl_bpart_get_kind(jl_binding_partition_t *bpart) JL_NOTSAFEPOINT { return (uint8_t)(bpart->kind & 0xf); } @@ -966,6 +976,7 @@ EXTERN_INLINE_DECLARE uint8_t jl_bpart_get_kind(jl_binding_partition_t *bpart) J STATIC_INLINE void jl_walk_binding_inplace(jl_binding_t **bnd, jl_binding_partition_t **bpart JL_PROPAGATES_ROOT, size_t world) JL_NOTSAFEPOINT; STATIC_INLINE void jl_walk_binding_inplace_depwarn(jl_binding_t **bnd, jl_binding_partition_t **bpart, size_t world, int *depwarn) JL_NOTSAFEPOINT; STATIC_INLINE void jl_walk_binding_inplace_all(jl_binding_t **bnd, jl_binding_partition_t **bpart JL_PROPAGATES_ROOT, int *depwarn, size_t min_world, size_t max_world) JL_NOTSAFEPOINT; +STATIC_INLINE void jl_walk_binding_inplace_worlds(jl_binding_t **bnd, jl_binding_partition_t **bpart, size_t *min_world, size_t *max_world, int *depwarn, size_t world) JL_NOTSAFEPOINT; #ifndef __clang_analyzer__ STATIC_INLINE void jl_walk_binding_inplace(jl_binding_t **bnd, jl_binding_partition_t **bpart, size_t world) JL_NOTSAFEPOINT @@ -1016,6 +1027,30 @@ STATIC_INLINE void jl_walk_binding_inplace_all(jl_binding_t **bnd, jl_binding_pa *bpart = jl_get_binding_partition_all(*bnd, min_world, max_world); } } + +STATIC_INLINE void jl_walk_binding_inplace_worlds(jl_binding_t **bnd, jl_binding_partition_t **bpart, size_t *min_world, size_t *max_world, int *depwarn, size_t world) JL_NOTSAFEPOINT +{ + int passed_explicit = 0; + while (*bpart) { + if (*min_world < (*bpart)->min_world) + *min_world = (*bpart)->min_world; + size_t bpart_max_world = jl_atomic_load_relaxed(&(*bpart)->max_world); + if (*max_world > bpart_max_world) + *max_world = bpart_max_world; + enum jl_partition_kind kind = jl_binding_kind(*bpart); + if (!jl_bkind_is_some_import(kind)) { + if (!passed_explicit && depwarn) + *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; + return; + } + if (!passed_explicit && depwarn) + *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; + if (kind != BINDING_KIND_IMPLICIT) + passed_explicit = 1; + *bnd = (jl_binding_t*)(*bpart)->restriction; + *bpart = jl_get_binding_partition(*bnd, world); + } +} #endif STATIC_INLINE int is10digit(char c) JL_NOTSAFEPOINT diff --git a/src/module.c b/src/module.c index 0e03601187964..6e3ffc12d2d5d 100644 --- a/src/module.c +++ b/src/module.c @@ -165,13 +165,9 @@ void jl_check_new_binding_implicit( return; } -STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED +STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, jl_value_t *parent, _Atomic(jl_binding_partition_t *)*insert, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED { - if (!b) - return NULL; assert(jl_is_binding(b)); - jl_value_t *parent = (jl_value_t*)b; - _Atomic(jl_binding_partition_t *)*insert = &b->partitions; jl_binding_partition_t *bpart = jl_atomic_load_relaxed(insert); size_t max_world = (size_t)-1; jl_binding_partition_t *new_bpart = NULL; @@ -202,12 +198,22 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b } jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b, size_t world) { + if (!b) + return NULL; // Duplicate the code for the entry frame for branch prediction - return jl_get_binding_partition_(b, world, NULL); + return jl_get_binding_partition_(b, (jl_value_t*)b, &b->partitions, world, NULL); +} + +jl_binding_partition_t *jl_get_binding_partition_with_hint(jl_binding_t *b, jl_binding_partition_t *prev, size_t world) JL_GLOBALLY_ROOTED { + // Helper for getting a binding partition for an older world after we've already looked up the partition for a newer world + assert(b); + assert(prev->min_world > world); + return jl_get_binding_partition_(b, (jl_value_t*)prev, &prev->next, world, NULL); } jl_binding_partition_t *jl_get_binding_partition2(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED { - return jl_get_binding_partition_(b, world, st); + assert(b); + return jl_get_binding_partition_(b, (jl_value_t*)b, &b->partitions, world, st); } jl_binding_partition_t *jl_get_binding_partition_all(jl_binding_t *b, size_t min_world, size_t max_world) { @@ -221,6 +227,53 @@ jl_binding_partition_t *jl_get_binding_partition_all(jl_binding_t *b, size_t min return bpart; } +JL_DLLEXPORT int jl_get_binding_leaf_partitions_restriction_kind(jl_binding_t *b JL_PROPAGATES_ROOT, struct restriction_kind_pair *rkp, size_t min_world, size_t max_world) { + if (!b) + return 0; + + int first = 1; + size_t validated_min_world = max_world == ~(size_t)0 ? ~(size_t)0 : max_world + 1; + jl_binding_partition_t *bpart = NULL; + int maybe_depwarn = 0; + while (validated_min_world > min_world) { + bpart = bpart ? jl_get_binding_partition_with_hint(b, bpart, validated_min_world - 1) : + jl_get_binding_partition(b, validated_min_world - 1); + while (validated_min_world > min_world && validated_min_world > bpart->min_world) { + jl_binding_t *curb = b; + jl_binding_partition_t *curbpart = bpart; + size_t cur_min_world = bpart->min_world; + size_t cur_max_world = validated_min_world - 1; + jl_walk_binding_inplace_worlds(&curb, &curbpart, &cur_min_world, &cur_max_world, &maybe_depwarn, cur_max_world); + if (first == 1) { + rkp->kind = jl_binding_kind(curbpart); + rkp->restriction = curbpart->restriction; + if (rkp->kind == BINDING_KIND_GLOBAL || rkp->kind == BINDING_KIND_DECLARED) + rkp->binding_if_global = curb; + first = 0; + } else { + if (jl_binding_kind(curbpart) != rkp->kind || curbpart->restriction != rkp->restriction) + return 0; + if ((rkp->kind == BINDING_KIND_GLOBAL || rkp->kind == BINDING_KIND_DECLARED) && rkp->binding_if_global != curb) + return 0; + } + validated_min_world = cur_min_world; + } + } + rkp->maybe_depwarn = maybe_depwarn; + return 1; +} + +JL_DLLEXPORT jl_value_t *jl_get_binding_leaf_partitions_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT, int *maybe_depwarn, size_t min_world, size_t max_world) { + struct restriction_kind_pair rkp = { NULL, NULL, BINDING_KIND_GUARD, 0 }; + if (!jl_get_binding_leaf_partitions_restriction_kind(b, &rkp, min_world, max_world)) + return NULL; + if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != BINDING_KIND_BACKDATED_CONST) { + *maybe_depwarn = rkp.maybe_depwarn; + return rkp.restriction; + } + return NULL; +} + JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) { jl_task_t *ct = jl_current_task; @@ -609,8 +662,13 @@ static inline void check_backdated_binding(jl_binding_t *b, enum jl_partition_ki JL_DLLEXPORT jl_value_t *jl_get_binding_value(jl_binding_t *b) { - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + return jl_get_binding_value_in_world(b, jl_current_task->world_age); +} + +JL_DLLEXPORT jl_value_t *jl_get_binding_value_in_world(jl_binding_t *b, size_t world) +{ + jl_binding_partition_t *bpart = jl_get_binding_partition(b, world); + jl_walk_binding_inplace(&b, &bpart, world); enum jl_partition_kind kind = jl_binding_kind(bpart); if (jl_bkind_is_some_guard(kind)) return NULL; diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index e70eb8dd97927..a0a0d4b1fa8d7 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -642,20 +642,17 @@ end function CC.abstract_eval_globalref(interp::REPLInterpreter, g::GlobalRef, bailed::Bool, sv::CC.InferenceState) # Ignore saw_latestworld - partition = CC.abstract_eval_binding_partition!(interp, g, sv) if (interp.limit_aggressive_inference ? is_repl_frame(sv) : is_call_graph_uncached(sv)) + partition = CC.abstract_eval_binding_partition!(interp, g, sv) if CC.is_defined_const_binding(CC.binding_kind(partition)) - return Pair{CC.RTEffects, Union{Nothing, Core.BindingPartition}}( - CC.RTEffects(Const(CC.partition_restriction(partition)), Union{}, CC.EFFECTS_TOTAL), partition) + return CC.RTEffects(Const(CC.partition_restriction(partition)), Union{}, CC.EFFECTS_TOTAL) else b = convert(Core.Binding, g) if CC.binding_kind(partition) == CC.BINDING_KIND_GLOBAL && isdefined(b, :value) - return Pair{CC.RTEffects, Union{Nothing, Core.BindingPartition}}( - CC.RTEffects(Const(b.value), Union{}, CC.EFFECTS_TOTAL), partition) + return CC.RTEffects(Const(b.value), Union{}, CC.EFFECTS_TOTAL) end end - return Pair{CC.RTEffects, Union{Nothing, Core.BindingPartition}}( - CC.RTEffects(Union{}, UndefVarError, CC.EFFECTS_THROWS), partition) + return CC.RTEffects(Union{}, UndefVarError, CC.EFFECTS_THROWS) end return @invoke CC.abstract_eval_globalref(interp::CC.AbstractInterpreter, g::GlobalRef, bailed::Bool, sv::CC.InferenceState) diff --git a/test/rebinding.jl b/test/rebinding.jl index e9dfa38261de5..b78d933ae9101 100644 --- a/test/rebinding.jl +++ b/test/rebinding.jl @@ -277,3 +277,24 @@ module ImageGlobalRefFlag @test Base.has_image_globalref(first(methods(fimage))) @test !Base.has_image_globalref(first(methods(fnoimage))) end + +# Test that inference can merge ranges for partitions as long as what's being imported doesn't change +module RangeMerge + using Test + using InteractiveUtils + + function get_llvm(@nospecialize(f), @nospecialize(t), raw=true, dump_module=false, optimize=true) + params = Base.CodegenParams(safepoint_on_entry=false, gcstack_arg = false, debug_info_level=Cint(2)) + d = InteractiveUtils._dump_function(f, t, false, false, raw, dump_module, :att, optimize, :none, false, params) + sprint(print, d) + end + + global x = 1 + const after_def_world = Base.get_world_counter() + export x + f() = x + @test f() == 1 + @test only(methods(f)).specializations.cache.min_world <= after_def_world + + @test !contains(get_llvm(f, Tuple{}), "jl_get_binding_value") +end From 7cf1db3a5eaf67f7d04b112b7558931711c3f4f9 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 3 Mar 2025 02:36:47 -0500 Subject: [PATCH 26/74] bpart: Rename partition flags, turn binding flags atomic (#57614) This renames the partition flags to start with PARTITION_ and turns the Binding flags into a bitfield that can be accessed atomically in preparation for adding further flags. (cherry picked from commit a802faf40f4fc8b3a98fb14422ac39d6293850c7) --- Compiler/src/Compiler.jl | 4 +- Compiler/src/abstractinterpretation.jl | 16 +-- base/invalidation.jl | 4 +- base/runtime_internals.jl | 46 ++++---- base/show.jl | 26 ++--- src/ast.c | 2 +- src/codegen.cpp | 14 +-- src/gf.c | 2 +- src/julia.h | 95 +++++++-------- src/julia_internal.h | 28 ++--- src/method.c | 4 +- src/module.c | 155 ++++++++++++------------- src/staticdata.c | 8 +- src/toplevel.c | 16 +-- stdlib/REPL/src/REPL.jl | 6 +- stdlib/REPL/src/REPLCompletions.jl | 2 +- stdlib/REPL/src/docview.jl | 2 +- test/rebinding.jl | 8 +- 18 files changed, 219 insertions(+), 219 deletions(-) diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index 490d988565ffd..c75a524ca40ec 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -49,8 +49,8 @@ using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstanc using Base using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospecializeinfer, - BINDING_KIND_GLOBAL, BINDING_KIND_UNDEF_CONST, BINDING_KIND_BACKDATED_CONST, BINDING_KIND_DECLARED, - BINDING_FLAG_DEPWARN, + PARTITION_KIND_GLOBAL, PARTITION_KIND_UNDEF_CONST, PARTITION_KIND_BACKDATED_CONST, PARTITION_KIND_DECLARED, + PARTITION_FLAG_DEPWARN, Base, BitVector, Bottom, Callable, DataTypeFieldDesc, EffectsOverride, Filter, Generator, IteratorSize, JLOptions, NUM_EFFECTS_OVERRIDES, OneTo, Ordering, RefValue, SizeUnknown, _NAMEDTUPLE_NAME, diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index d733f79f949bf..65443bef73c51 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -2441,7 +2441,7 @@ end (valid_worlds, rt) = scan_leaf_partitions(interp, gr, sv.world) do interp, _, partition local rt kind = binding_kind(partition) - if is_some_guard(kind) || kind == BINDING_KIND_DECLARED + if is_some_guard(kind) || kind == PARTITION_KIND_DECLARED # We do not currently assume an invalidation for guard -> defined transitions # rt = Const(nothing) rt = Type @@ -2572,7 +2572,7 @@ function abstract_eval_replaceglobal!(interp::AbstractInterpreter, sv::AbsIntSta (valid_worlds, (rte, T)) = scan_leaf_partitions(interp, gr, sv.world) do interp, _, partition partition_T = nothing partition_rte = abstract_eval_partition_load(interp, partition) - if binding_kind(partition) == BINDING_KIND_GLOBAL + if binding_kind(partition) == PARTITION_KIND_GLOBAL partition_T = partition_restriction(partition) end partition_exct = Union{partition_rte.exct, global_assignment_binding_rt_exct(interp, partition, v)[2]} @@ -3513,9 +3513,9 @@ abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, ::Core abstract_eval_partition_load(interp, partition) function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing}, partition::Core.BindingPartition) kind = binding_kind(partition) - isdepwarn = (partition.kind & BINDING_FLAG_DEPWARN) != 0 + isdepwarn = (partition.kind & PARTITION_FLAG_DEPWARN) != 0 local_getglobal_effects = Effects(generic_getglobal_effects, effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE) - if is_some_guard(kind) || kind == BINDING_KIND_UNDEF_CONST + if is_some_guard(kind) || kind == PARTITION_KIND_UNDEF_CONST if interp !== nothing && InferenceParams(interp).assume_bindings_static return RTEffects(Union{}, UndefVarError, EFFECTS_THROWS) else @@ -3526,7 +3526,7 @@ function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing end if is_defined_const_binding(kind) - if kind == BINDING_KIND_BACKDATED_CONST + if kind == PARTITION_KIND_BACKDATED_CONST # Infer this as guard. We do not want a later const definition to retroactively improve # inference results in an earlier world. return RTEffects(Any, UndefVarError, local_getglobal_effects) @@ -3537,7 +3537,7 @@ function abstract_eval_partition_load(interp::Union{AbstractInterpreter, Nothing effect_free=isdepwarn ? ALWAYS_FALSE : ALWAYS_TRUE)) end - if kind == BINDING_KIND_DECLARED + if kind == PARTITION_KIND_DECLARED # Could be replaced by a backdated const which has an effect, so we can't assume it won't. # Besides, we would prefer not to merge the world range for this into the world range for # _GLOBAL, because that would pessimize codegen. @@ -3604,7 +3604,7 @@ function abstract_eval_globalref(interp, g::GlobalRef, saw_latestworld::Bool, sv # binding we end up reaching such that codegen can emit a simpler pointer load. Pair{RTEffects, Union{Nothing, Core.Binding}}( abstract_eval_partition_load(interp, partition), - binding_kind(partition) in (BINDING_KIND_GLOBAL, BINDING_KIND_DECLARED) ? binding : nothing) + binding_kind(partition) in (PARTITION_KIND_GLOBAL, PARTITION_KIND_DECLARED) ? binding : nothing) end update_valid_age!(sv, valid_worlds) if ret.rt !== Union{} && ret.exct === UndefVarError && binding_if_global !== nothing && InferenceParams(interp).assume_bindings_static @@ -3633,7 +3633,7 @@ function global_assignment_binding_rt_exct(interp::AbstractInterpreter, partitio elseif is_some_const_binding(kind) || is_some_imported(kind) return Pair{Any,Any}(Bottom, ErrorException) end - ty = kind == BINDING_KIND_DECLARED ? Any : partition_restriction(partition) + ty = kind == PARTITION_KIND_DECLARED ? Any : partition_restriction(partition) wnewty = widenconst(newty) if !hasintersect(wnewty, ty) return Pair{Any,Any}(Bottom, TypeError) diff --git a/base/invalidation.jl b/base/invalidation.jl index fbf54cc2e24a6..539999c15dd1d 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -140,7 +140,7 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core end end end - if (invalidated_bpart.kind & BINDING_FLAG_EXPORTED != 0) || (new_bpart !== nothing && (new_bpart.kind & BINDING_FLAG_EXPORTED != 0)) + if (invalidated_bpart.kind & PARTITION_FLAG_EXPORTED != 0) || (new_bpart !== nothing && (new_bpart.kind & PARTITION_FLAG_EXPORTED != 0)) # This binding was exported - we need to check all modules that `using` us to see if they # have a binding that is affected by this change. usings_backedges = ccall(:jl_get_module_usings_backedges, Any, (Any,), gr.mod) @@ -151,7 +151,7 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core isdefined(user_binding, :partitions) || continue latest_bpart = user_binding.partitions latest_bpart.max_world == typemax(UInt) || continue - binding_kind(latest_bpart) in (BINDING_KIND_IMPLICIT, BINDING_KIND_FAILED, BINDING_KIND_GUARD) || continue + binding_kind(latest_bpart) in (PARTITION_KIND_IMPLICIT, PARTITION_KIND_FAILED, PARTITION_KIND_GUARD) || continue @atomic :release latest_bpart.max_world = new_max_world invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, nothing, new_max_world) end diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 6850859a7ba98..536705f1bdc2a 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -197,29 +197,29 @@ function _fieldnames(@nospecialize t) end # N.B.: Needs to be synced with julia.h -const BINDING_KIND_CONST = 0x0 -const BINDING_KIND_CONST_IMPORT = 0x1 -const BINDING_KIND_GLOBAL = 0x2 -const BINDING_KIND_IMPLICIT = 0x3 -const BINDING_KIND_EXPLICIT = 0x4 -const BINDING_KIND_IMPORTED = 0x5 -const BINDING_KIND_FAILED = 0x6 -const BINDING_KIND_DECLARED = 0x7 -const BINDING_KIND_GUARD = 0x8 -const BINDING_KIND_UNDEF_CONST = 0x9 -const BINDING_KIND_BACKDATED_CONST = 0xa - -const BINDING_FLAG_EXPORTED = 0x10 -const BINDING_FLAG_DEPRECATED = 0x20 -const BINDING_FLAG_DEPWARN = 0x40 - -const BINDING_KIND_MASK = 0x0f -const BINDING_FLAG_MASK = 0xf0 - -is_defined_const_binding(kind::UInt8) = (kind == BINDING_KIND_CONST || kind == BINDING_KIND_CONST_IMPORT || kind == BINDING_KIND_BACKDATED_CONST) -is_some_const_binding(kind::UInt8) = (is_defined_const_binding(kind) || kind == BINDING_KIND_UNDEF_CONST) -is_some_imported(kind::UInt8) = (kind == BINDING_KIND_IMPLICIT || kind == BINDING_KIND_EXPLICIT || kind == BINDING_KIND_IMPORTED) -is_some_guard(kind::UInt8) = (kind == BINDING_KIND_GUARD || kind == BINDING_KIND_FAILED || kind == BINDING_KIND_UNDEF_CONST) +const PARTITION_KIND_CONST = 0x0 +const PARTITION_KIND_CONST_IMPORT = 0x1 +const PARTITION_KIND_GLOBAL = 0x2 +const PARTITION_KIND_IMPLICIT = 0x3 +const PARTITION_KIND_EXPLICIT = 0x4 +const PARTITION_KIND_IMPORTED = 0x5 +const PARTITION_KIND_FAILED = 0x6 +const PARTITION_KIND_DECLARED = 0x7 +const PARTITION_KIND_GUARD = 0x8 +const PARTITION_KIND_UNDEF_CONST = 0x9 +const PARTITION_KIND_BACKDATED_CONST = 0xa + +const PARTITION_FLAG_EXPORTED = 0x10 +const PARTITION_FLAG_DEPRECATED = 0x20 +const PARTITION_FLAG_DEPWARN = 0x40 + +const PARTITION_MASK_KIND = 0x0f +const PARTITION_MASK_FLAG = 0xf0 + +is_defined_const_binding(kind::UInt8) = (kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_BACKDATED_CONST) +is_some_const_binding(kind::UInt8) = (is_defined_const_binding(kind) || kind == PARTITION_KIND_UNDEF_CONST) +is_some_imported(kind::UInt8) = (kind == PARTITION_KIND_IMPLICIT || kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) +is_some_guard(kind::UInt8) = (kind == PARTITION_KIND_GUARD || kind == PARTITION_KIND_FAILED || kind == PARTITION_KIND_UNDEF_CONST) function lookup_binding_partition(world::UInt, b::Core.Binding) ccall(:jl_get_binding_partition, Ref{Core.BindingPartition}, (Any, UInt), b, world) diff --git a/base/show.jl b/base/show.jl index f1336687fbef5..055f0923d9432 100644 --- a/base/show.jl +++ b/base/show.jl @@ -3370,17 +3370,17 @@ function print_partition(io::IO, partition::Core.BindingPartition) else print(io, max_world) end - if (partition.kind & BINDING_FLAG_MASK) != 0 + if (partition.kind & PARTITION_MASK_FLAG) != 0 first = false print(io, " [") - if (partition.kind & BINDING_FLAG_EXPORTED) != 0 + if (partition.kind & PARTITION_FLAG_EXPORTED) != 0 print(io, "exported") end - if (partition.kind & BINDING_FLAG_DEPRECATED) != 0 + if (partition.kind & PARTITION_FLAG_DEPRECATED) != 0 first ? (first = false) : print(io, ",") print(io, "deprecated") end - if (partition.kind & BINDING_FLAG_DEPWARN) != 0 + if (partition.kind & PARTITION_FLAG_DEPWARN) != 0 first ? (first = false) : print(io, ",") print(io, "depwarn") end @@ -3388,31 +3388,31 @@ function print_partition(io::IO, partition::Core.BindingPartition) end print(io, " - ") kind = binding_kind(partition) - if kind == BINDING_KIND_BACKDATED_CONST + if kind == PARTITION_KIND_BACKDATED_CONST print(io, "backdated constant binding to ") print(io, partition_restriction(partition)) elseif is_defined_const_binding(kind) print(io, "constant binding to ") print(io, partition_restriction(partition)) - elseif kind == BINDING_KIND_UNDEF_CONST + elseif kind == PARTITION_KIND_UNDEF_CONST print(io, "undefined const binding") - elseif kind == BINDING_KIND_GUARD + elseif kind == PARTITION_KIND_GUARD print(io, "undefined binding - guard entry") - elseif kind == BINDING_KIND_FAILED + elseif kind == PARTITION_KIND_FAILED print(io, "ambiguous binding - guard entry") - elseif kind == BINDING_KIND_DECLARED + elseif kind == PARTITION_KIND_DECLARED print(io, "weak global binding declared using `global` (implicit type Any)") - elseif kind == BINDING_KIND_IMPLICIT + elseif kind == PARTITION_KIND_IMPLICIT print(io, "implicit `using` from ") print(io, partition_restriction(partition).globalref) - elseif kind == BINDING_KIND_EXPLICIT + elseif kind == PARTITION_KIND_EXPLICIT print(io, "explicit `using` from ") print(io, partition_restriction(partition).globalref) - elseif kind == BINDING_KIND_IMPORTED + elseif kind == PARTITION_KIND_IMPORTED print(io, "explicit `import` from ") print(io, partition_restriction(partition).globalref) else - @assert kind == BINDING_KIND_GLOBAL + @assert kind == PARTITION_KIND_GLOBAL print(io, "global variable with type ") print(io, partition_restriction(partition)) end diff --git a/src/ast.c b/src/ast.c index eeff17162f3fc..ab6b04efa526a 100644 --- a/src/ast.c +++ b/src/ast.c @@ -178,7 +178,7 @@ static value_t fl_defined_julia_global(fl_context_t *fl_ctx, value_t *args, uint jl_sym_t *var = scmsym_to_julia(fl_ctx, args[0]); jl_binding_t *b = jl_get_module_binding(ctx->module, var, 0); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - return (bpart != NULL && jl_binding_kind(bpart) == BINDING_KIND_GLOBAL) ? fl_ctx->T : fl_ctx->F; + return (bpart != NULL && jl_binding_kind(bpart) == PARTITION_KIND_GLOBAL) ? fl_ctx->T : fl_ctx->F; } // Used to generate a unique suffix for a given symbol (e.g. variable or type name) diff --git a/src/codegen.cpp b/src/codegen.cpp index d70dcc227847d..8e524c9d3ba2d 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -3237,11 +3237,11 @@ static jl_cgval_t emit_globalref_runtime(jl_codectx_t &ctx, jl_binding_t *bnd, j static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *name, AtomicOrdering order) { jl_binding_t *bnd = jl_get_module_binding(mod, name, 1); - struct restriction_kind_pair rkp = { NULL, NULL, BINDING_KIND_GUARD, 0 }; + struct restriction_kind_pair rkp = { NULL, NULL, PARTITION_KIND_GUARD, 0 }; if (!jl_get_binding_leaf_partitions_restriction_kind(bnd, &rkp, ctx.min_world, ctx.max_world)) { return emit_globalref_runtime(ctx, bnd, mod, name); } - if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != BINDING_KIND_BACKDATED_CONST) { + if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != PARTITION_KIND_BACKDATED_CONST) { if (rkp.maybe_depwarn) { Value *bp = julia_binding_gv(ctx, bnd); ctx.builder.CreateCall(prepare_call(jldepcheck_func), { bp }); @@ -3253,7 +3253,7 @@ static jl_cgval_t emit_globalref(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t * } return mark_julia_const(ctx, constval); } - if (rkp.kind != BINDING_KIND_GLOBAL) { + if (rkp.kind != PARTITION_KIND_GLOBAL) { return emit_globalref_runtime(ctx, bnd, mod, name); } Value *bp = julia_binding_gv(ctx, bnd); @@ -3278,8 +3278,8 @@ static jl_cgval_t emit_globalop(jl_codectx_t &ctx, jl_module_t *mod, jl_sym_t *s jl_binding_partition_t *bpart = jl_get_binding_partition_all(bnd, ctx.min_world, ctx.max_world); Value *bp = julia_binding_gv(ctx, bnd); if (bpart) { - if (jl_binding_kind(bpart) == BINDING_KIND_GLOBAL) { - int possibly_deprecated = bpart->kind & BINDING_FLAG_DEPWARN; + if (jl_binding_kind(bpart) == PARTITION_KIND_GLOBAL) { + int possibly_deprecated = bpart->kind & PARTITION_FLAG_DEPWARN; jl_value_t *ty = bpart->restriction; if (ty != nullptr) { const std::string fname = issetglobal ? "setglobal!" : isreplaceglobal ? "replaceglobal!" : isswapglobal ? "swapglobal!" : ismodifyglobal ? "modifyglobal!" : "setglobalonce!"; @@ -3826,11 +3826,11 @@ static jl_cgval_t emit_isdefinedglobal(jl_codectx_t &ctx, jl_module_t *modu, jl_ { Value *isnull = NULL; jl_binding_t *bnd = allow_import ? jl_get_binding(modu, name) : jl_get_module_binding(modu, name, 0); - struct restriction_kind_pair rkp = { NULL, NULL, BINDING_KIND_GUARD, 0 }; + struct restriction_kind_pair rkp = { NULL, NULL, PARTITION_KIND_GUARD, 0 }; if (allow_import && jl_get_binding_leaf_partitions_restriction_kind(bnd, &rkp, ctx.min_world, ctx.max_world)) { if (jl_bkind_is_some_constant(rkp.kind)) return mark_julia_const(ctx, rkp.restriction); - if (rkp.kind == BINDING_KIND_GLOBAL) { + if (rkp.kind == PARTITION_KIND_GLOBAL) { Value *bp = julia_binding_gv(ctx, rkp.binding_if_global); bp = julia_binding_pvalue(ctx, bp); LoadInst *v = ctx.builder.CreateAlignedLoad(ctx.types().T_prjlvalue, bp, Align(sizeof(void*))); diff --git a/src/gf.c b/src/gf.c index 82e1e43333eb4..1d93d03cc59d2 100644 --- a/src/gf.c +++ b/src/gf.c @@ -3816,7 +3816,7 @@ jl_function_t *jl_new_generic_function_with_supertype(jl_sym_t *name, jl_module_ JL_GC_PUSH1(&ftype); ftype->name->mt->name = name; jl_gc_wb(ftype->name->mt, name); - jl_declare_constant_val3(NULL, module, tname, (jl_value_t*)ftype, BINDING_KIND_CONST, new_world); + jl_declare_constant_val3(NULL, module, tname, (jl_value_t*)ftype, PARTITION_KIND_CONST, new_world); jl_value_t *f = jl_new_struct(ftype); ftype->instance = f; jl_gc_wb(ftype, f); diff --git a/src/julia.h b/src/julia.h index 8ee57acbabb4d..4d311d84366d9 100644 --- a/src/julia.h +++ b/src/julia.h @@ -623,27 +623,27 @@ typedef struct _jl_weakref_t { // These binding kinds depend solely on the set of using'd packages and are not explicitly // declared: // -// BINDING_KIND_IMPLICIT -// BINDING_KIND_GUARD -// BINDING_KIND_FAILED +// PARTITION_KIND_IMPLICIT +// PARTITION_KIND_GUARD +// PARTITION_KIND_FAILED // // 2. Weakly Declared Bindings (Weak) // The binding was declared using `global`. It is treated as a mutable, `Any` type global // for almost all purposes, except that it receives slightly worse optimizations, since it // may be replaced. // -// BINDING_KIND_DECLARED +// PARTITION_KIND_DECLARED // // 3. Strong Declared Bindings (Weak) // All other bindings are explicitly declared using a keyword or global assignment. // These are considered strongest: // -// BINDING_KIND_CONST -// BINDING_KIND_CONST_IMPORT -// BINDING_KIND_EXPLICIT -// BINDING_KIND_IMPORTED -// BINDING_KIND_GLOBAL -// BINDING_KIND_UNDEF_CONST +// PARTITION_KIND_CONST +// PARTITION_KIND_CONST_IMPORT +// PARTITION_KIND_EXPLICIT +// PARTITION_KIND_IMPORTED +// PARTITION_KIND_GLOBAL +// PARTITION_KIND_UNDEF_CONST // // The runtime supports syntactic invalidation (by raising the world age and changing the partition type // in the new world age) from any partition kind to any other. @@ -651,86 +651,86 @@ typedef struct _jl_weakref_t { // However, not all transitions are allowed syntactically. We have the following rules for SYNTACTIC invalidation: // 1. It is always syntactically permissable to replace a weaker binding by a stronger binding // 2. Implicit bindings can be syntactically changed to other implicit bindings by changing the `using` set. -// 3. Finally, we syntactically permit replacing one BINDING_KIND_CONST(_IMPORT) by another of a different value. +// 3. Finally, we syntactically permit replacing one PARTITION_KIND_CONST(_IMPORT) by another of a different value. // // We may make this list more permissive in the future. // -// Finally, BINDING_KIND_BACKDATED_CONST is a special case, and the only case where we may replace an +// Finally, PARTITION_KIND_BACKDATED_CONST is a special case, and the only case where we may replace an // existing partition by a different partition kind in the same world age. As such, it needs special -// support in inference. Any partition kind that may be replaced by a BINDING_KIND_BACKDATED_CONST -// must be inferred accordingly. BINDING_KIND_BACKDATED_CONST is intended as a temporary compatibility -// measure. The following kinds may be replaced by BINDING_KIND_BACKDATED_CONST: -// - BINDING_KIND_GUARD -// - BINDING_KIND_FAILED -// - BINDING_KIND_DECLARED +// support in inference. Any partition kind that may be replaced by a PARTITION_KIND_BACKDATED_CONST +// must be inferred accordingly. PARTITION_KIND_BACKDATED_CONST is intended as a temporary compatibility +// measure. The following kinds may be replaced by PARTITION_KIND_BACKDATED_CONST: +// - PARTITION_KIND_GUARD +// - PARTITION_KIND_FAILED +// - PARTITION_KIND_DECLARED enum jl_partition_kind { // Constant: This binding partition is a constant declared using `const _ = ...` // ->restriction holds the constant value - BINDING_KIND_CONST = 0x0, + PARTITION_KIND_CONST = 0x0, // Import Constant: This binding partition is a constant declared using `import A` // ->restriction holds the constant value - BINDING_KIND_CONST_IMPORT = 0x1, + PARTITION_KIND_CONST_IMPORT = 0x1, // Global: This binding partition is a global variable. It was declared either using // `global x::T` to implicitly through a syntactic global assignment. // -> restriction holds the type restriction - BINDING_KIND_GLOBAL = 0x2, + PARTITION_KIND_GLOBAL = 0x2, // Implicit: The binding was implicitly imported from a `using`'d module. // ->restriction holds the imported binding - BINDING_KIND_IMPLICIT = 0x3, + PARTITION_KIND_IMPLICIT = 0x3, // Explicit: The binding was explicitly `using`'d by name // ->restriction holds the imported binding - BINDING_KIND_EXPLICIT = 0x4, + PARTITION_KIND_EXPLICIT = 0x4, // Imported: The binding was explicitly `import`'d by name // ->restriction holds the imported binding - BINDING_KIND_IMPORTED = 0x5, + PARTITION_KIND_IMPORTED = 0x5, // Failed: We attempted to import the binding, but the import was ambiguous // ->restriction is NULL. - BINDING_KIND_FAILED = 0x6, + PARTITION_KIND_FAILED = 0x6, // Declared: The binding was declared using `global` or similar. This acts in most ways like - // BINDING_KIND_GLOBAL with an `Any` restriction, except that it may be redefined to a stronger + // PARTITION_KIND_GLOBAL with an `Any` restriction, except that it may be redefined to a stronger // binding like `const` or an explicit import. // ->restriction is NULL. - BINDING_KIND_DECLARED = 0x7, + PARTITION_KIND_DECLARED = 0x7, // Guard: The binding was looked at, but no global or import was resolved at the time // ->restriction is NULL. - BINDING_KIND_GUARD = 0x8, + PARTITION_KIND_GUARD = 0x8, // Undef Constant: This binding partition is a constant declared using `const`, but // without a value. // ->restriction is NULL - BINDING_KIND_UNDEF_CONST = 0x9, + PARTITION_KIND_UNDEF_CONST = 0x9, // Backated constant. A constant that was backdated for compatibility. In all other - // ways equivalent to BINDING_KIND_CONST, but prints a warning on access - BINDING_KIND_BACKDATED_CONST = 0xa, + // ways equivalent to PARTITION_KIND_CONST, but prints a warning on access + PARTITION_KIND_BACKDATED_CONST = 0xa, // This is not a real binding kind, but can be used to ask for a re-resolution // of the implicit binding kind - BINDING_KIND_IMPLICIT_RECOMPUTE = 0xb + PARTITION_KIND_IMPLICIT_RECOMPUTE = 0xb }; -static const uint8_t BINDING_KIND_MASK = 0x0f; -static const uint8_t BINDING_FLAG_MASK = 0xf0; +static const uint8_t PARTITION_MASK_KIND = 0x0f; +static const uint8_t PARTITION_MASK_FLAG = 0xf0; //// These are flags that get anded into the above // // _EXPORTED: This binding partition is exported. In the world ranges covered by this partitions, // other modules that `using` this module, may implicit import this binding. -static const uint8_t BINDING_FLAG_EXPORTED = 0x10; +static const uint8_t PARTITION_FLAG_EXPORTED = 0x10; // _DEPRECATED: This binding partition is deprecated. It is considered weak for the purposes of // implicit import resolution. -static const uint8_t BINDING_FLAG_DEPRECATED = 0x20; +static const uint8_t PARTITION_FLAG_DEPRECATED = 0x20; // _DEPWARN: This binding partition will print a deprecation warning on access. Note that _DEPWARN // implies _DEPRECATED. However, the reverse is not true. Such bindings are usually used for functions, // where calling the function itself will provide a (better) deprecation warning/error. -static const uint8_t BINDING_FLAG_DEPWARN = 0x40; +static const uint8_t PARTITION_FLAG_DEPWARN = 0x40; typedef struct __attribute__((aligned(8))) _jl_binding_partition_t { JL_DATA_TYPE /* union { - * // For ->kind == BINDING_KIND_GLOBAL + * // For ->kind == PARTITION_KIND_GLOBAL * jl_value_t *type_restriction; - * // For ->kind == BINDING_KIND_CONST(_IMPORT) + * // For ->kind == PARTITION_KIND_CONST(_IMPORT) * jl_value_t *constval; - * // For ->kind in (BINDING_KIND_IMPLICIT, BINDING_KIND_EXPLICIT, BINDING_KIND_IMPORT) + * // For ->kind in (PARTITION_KIND_IMPLICIT, PARTITION_KIND_EXPLICIT, PARTITION_KIND_IMPORT) * jl_binding_t *imported; * } restriction; */ @@ -746,17 +746,20 @@ STATIC_INLINE enum jl_partition_kind jl_binding_kind(jl_binding_partition_t *bpa return (enum jl_partition_kind)(bpart->kind & 0xf); } +enum jl_binding_flags { + BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION = 0x1, + BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION = 0x2, + // `export` is tracked in partitions, but sets this as well + BINDING_FLAG_PUBLICP = 0x4 +}; + typedef struct _jl_binding_t { JL_DATA_TYPE jl_globalref_t *globalref; // cached GlobalRef for this binding _Atomic(jl_value_t*) value; _Atomic(jl_binding_partition_t*) partitions; jl_array_t *backedges; - uint8_t did_print_backdate_admonition:1; - uint8_t did_print_implicit_import_admonition:1; - uint8_t publicp:1; // `export` is tracked in partitions, but sets this as well - uint8_t deprecated:2; // 0=not deprecated, 1=renamed, 2=moved to another package - uint8_t padding:3; + _Atomic(uint8_t) flags; } jl_binding_t; typedef struct { @@ -784,7 +787,7 @@ typedef struct _jl_module_t { int8_t infer; uint8_t istopmod; int8_t max_methods; - // If cleared no binding partition in this module has BINDING_FLAG_EXPORTED and min_world > jl_require_world. + // If cleared no binding partition in this module has PARTITION_FLAG_EXPORTED and min_world > jl_require_world. _Atomic(int8_t) export_set_changed_since_require_world; jl_mutex_t lock; intptr_t hash; diff --git a/src/julia_internal.h b/src/julia_internal.h index 315cc24c9f7a9..ff11926fe42c9 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -937,23 +937,23 @@ jl_method_t *jl_make_opaque_closure_method(jl_module_t *module, jl_value_t *name JL_DLLEXPORT int jl_is_valid_oc_argtype(jl_tupletype_t *argt, jl_method_t *source); STATIC_INLINE int jl_bkind_is_some_import(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == BINDING_KIND_IMPLICIT || kind == BINDING_KIND_EXPLICIT || kind == BINDING_KIND_IMPORTED; + return kind == PARTITION_KIND_IMPLICIT || kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED; } STATIC_INLINE int jl_bkind_is_some_guard(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == BINDING_KIND_FAILED || kind == BINDING_KIND_GUARD; + return kind == PARTITION_KIND_FAILED || kind == PARTITION_KIND_GUARD; } STATIC_INLINE int jl_bkind_is_some_implicit(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == BINDING_KIND_IMPLICIT || jl_bkind_is_some_guard(kind); + return kind == PARTITION_KIND_IMPLICIT || jl_bkind_is_some_guard(kind); } STATIC_INLINE int jl_bkind_is_some_constant(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == BINDING_KIND_CONST || kind == BINDING_KIND_CONST_IMPORT || kind == BINDING_KIND_UNDEF_CONST || kind == BINDING_KIND_BACKDATED_CONST; + return kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_UNDEF_CONST || kind == PARTITION_KIND_BACKDATED_CONST; } STATIC_INLINE int jl_bkind_is_defined_constant(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == BINDING_KIND_CONST || kind == BINDING_KIND_CONST_IMPORT || kind == BINDING_KIND_BACKDATED_CONST; + return kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_BACKDATED_CONST; } JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world) JL_GLOBALLY_ROOTED; @@ -996,12 +996,12 @@ STATIC_INLINE void jl_walk_binding_inplace_depwarn(jl_binding_t **bnd, jl_bindin enum jl_partition_kind kind = jl_binding_kind(*bpart); if (!jl_bkind_is_some_import(kind)) { if (!passed_explicit && depwarn) - *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; + *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; return; } if (!passed_explicit && depwarn) - *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; - if (kind != BINDING_KIND_IMPLICIT) + *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; + if (kind != PARTITION_KIND_IMPLICIT) passed_explicit = 1; *bnd = (jl_binding_t*)(*bpart)->restriction; *bpart = jl_get_binding_partition(*bnd, world); @@ -1016,12 +1016,12 @@ STATIC_INLINE void jl_walk_binding_inplace_all(jl_binding_t **bnd, jl_binding_pa enum jl_partition_kind kind = jl_binding_kind(*bpart); if (!jl_bkind_is_some_import(kind)) { if (!passed_explicit && depwarn) - *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; + *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; return; } if (!passed_explicit && depwarn) - *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; - if (kind != BINDING_KIND_IMPLICIT) + *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; + if (kind != PARTITION_KIND_IMPLICIT) passed_explicit = 1; *bnd = (jl_binding_t*)(*bpart)->restriction; *bpart = jl_get_binding_partition_all(*bnd, min_world, max_world); @@ -1040,12 +1040,12 @@ STATIC_INLINE void jl_walk_binding_inplace_worlds(jl_binding_t **bnd, jl_binding enum jl_partition_kind kind = jl_binding_kind(*bpart); if (!jl_bkind_is_some_import(kind)) { if (!passed_explicit && depwarn) - *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; + *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; return; } if (!passed_explicit && depwarn) - *depwarn |= (*bpart)->kind & BINDING_FLAG_DEPWARN; - if (kind != BINDING_KIND_IMPLICIT) + *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; + if (kind != PARTITION_KIND_IMPLICIT) passed_explicit = 1; *bnd = (jl_binding_t*)(*bpart)->restriction; *bpart = jl_get_binding_partition(*bnd, world); diff --git a/src/method.c b/src/method.c index 1fd36b102e0f6..ffd33645310f3 100644 --- a/src/method.c +++ b/src/method.c @@ -1069,7 +1069,7 @@ JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name) jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); jl_value_t *gf = NULL; enum jl_partition_kind kind = jl_binding_kind(bpart); - if (!jl_bkind_is_some_guard(kind) && kind != BINDING_KIND_DECLARED && kind != BINDING_KIND_IMPLICIT) { + if (!jl_bkind_is_some_guard(kind) && kind != PARTITION_KIND_DECLARED && kind != PARTITION_KIND_IMPLICIT) { jl_walk_binding_inplace(&b, &bpart, new_world); if (jl_bkind_is_some_constant(jl_binding_kind(bpart))) { gf = bpart->restriction; @@ -1084,7 +1084,7 @@ JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name) // From this point on (if we didn't error), we're committed to raising the world age, // because we've used it to declare the type name. jl_atomic_store_release(&jl_world_counter, new_world); - jl_declare_constant_val3(b, mod, name, gf, BINDING_KIND_CONST, new_world); + jl_declare_constant_val3(b, mod, name, gf, PARTITION_KIND_CONST, new_world); JL_GC_PROMISE_ROOTED(gf); JL_UNLOCK(&world_counter_lock); return gf; diff --git a/src/module.c b/src/module.c index 6e3ffc12d2d5d..8f5af51da7668 100644 --- a/src/module.c +++ b/src/module.c @@ -19,7 +19,7 @@ static jl_binding_partition_t *new_binding_partition(void) { jl_binding_partition_t *bpart = (jl_binding_partition_t*)jl_gc_alloc(jl_current_task->ptls, sizeof(jl_binding_partition_t), jl_binding_partition_type); bpart->restriction = NULL; - bpart->kind = (size_t)BINDING_KIND_GUARD; + bpart->kind = (size_t)PARTITION_KIND_GUARD; bpart->min_world = 0; jl_atomic_store_relaxed(&bpart->max_world, (size_t)-1); jl_atomic_store_relaxed(&bpart->next, NULL); @@ -54,7 +54,7 @@ void jl_check_new_binding_implicit( for (; tmp != NULL; tmp = tmp->prev) { if (tmp->b == b) { new_bpart->restriction = NULL; - new_bpart->kind = BINDING_KIND_FAILED; /* BINDING_KIND_CYCLE */ + new_bpart->kind = PARTITION_KIND_FAILED; /* PARTITION_KIND_CYCLE */ return; } } @@ -73,7 +73,7 @@ void jl_check_new_binding_implicit( JL_LOCK(&m->lock); int i = (int)module_usings_length(m) - 1; JL_UNLOCK(&m->lock); - enum jl_partition_kind guard_kind = BINDING_KIND_GUARD; + enum jl_partition_kind guard_kind = PARTITION_KIND_GUARD; for (; i >= 0; --i) { JL_LOCK(&m->lock); struct _jl_module_using data = *module_usings_getidx(m, i); @@ -106,16 +106,16 @@ void jl_check_new_binding_implicit( if (tempbmax_world < max_world) max_world = tempbmax_world; - if ((tempbpart->kind & BINDING_FLAG_EXPORTED) == 0) + if ((tempbpart->kind & PARTITION_FLAG_EXPORTED) == 0) continue; if (impb) { - if (tempbpart->kind & BINDING_FLAG_DEPRECATED) + if (tempbpart->kind & PARTITION_FLAG_DEPRECATED) continue; - if (jl_binding_kind(tempbpart) == BINDING_KIND_GUARD && - jl_binding_kind(impbpart) != BINDING_KIND_GUARD) + if (jl_binding_kind(tempbpart) == PARTITION_KIND_GUARD && + jl_binding_kind(impbpart) != PARTITION_KIND_GUARD) continue; - if (jl_binding_kind(impbpart) == BINDING_KIND_GUARD) { + if (jl_binding_kind(impbpart) == PARTITION_KIND_GUARD) { impb = tempb; impbpart = tempbpart; continue; @@ -125,17 +125,17 @@ void jl_check_new_binding_implicit( // Binding is ambiguous // TODO: Even for eq bindings, this may need to further constrain the world age. deprecated_impb = impb = NULL; - guard_kind = BINDING_KIND_FAILED; + guard_kind = PARTITION_KIND_FAILED; break; } - else if (tempbpart->kind & BINDING_FLAG_DEPRECATED) { + else if (tempbpart->kind & PARTITION_FLAG_DEPRECATED) { if (deprecated_impb) { if (!eq_bindings(tempbpart, deprecated_impb, world)) { - guard_kind = BINDING_KIND_FAILED; + guard_kind = PARTITION_KIND_FAILED; deprecated_impb = NULL; } } - else if (guard_kind == BINDING_KIND_GUARD) { + else if (guard_kind == PARTITION_KIND_GUARD) { deprecated_impb = tempb; } } @@ -153,7 +153,7 @@ void jl_check_new_binding_implicit( new_bpart->min_world = min_world; jl_atomic_store_relaxed(&new_bpart->max_world, max_world); if (impb) { - new_bpart->kind = BINDING_KIND_IMPLICIT; + new_bpart->kind = PARTITION_KIND_IMPLICIT; new_bpart->restriction = (jl_value_t*)impb; jl_gc_wb(new_bpart, impb); // TODO: World age constraints? @@ -188,8 +188,8 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b jl_atomic_store_relaxed(&new_bpart->max_world, max_world); JL_GC_PROMISE_ROOTED(new_bpart); // TODO: Analyzer doesn't understand MAYBE_UNROOTED properly jl_check_new_binding_implicit(new_bpart, b, st, world); - if (bpart && (bpart->kind & BINDING_FLAG_EXPORTED)) - new_bpart->kind |= BINDING_FLAG_EXPORTED; + if (bpart && (bpart->kind & PARTITION_FLAG_EXPORTED)) + new_bpart->kind |= PARTITION_FLAG_EXPORTED; if (jl_atomic_cmpswap(insert, &bpart, new_bpart)) { jl_gc_wb(parent, new_bpart); return new_bpart; @@ -247,13 +247,13 @@ JL_DLLEXPORT int jl_get_binding_leaf_partitions_restriction_kind(jl_binding_t *b if (first == 1) { rkp->kind = jl_binding_kind(curbpart); rkp->restriction = curbpart->restriction; - if (rkp->kind == BINDING_KIND_GLOBAL || rkp->kind == BINDING_KIND_DECLARED) + if (rkp->kind == PARTITION_KIND_GLOBAL || rkp->kind == PARTITION_KIND_DECLARED) rkp->binding_if_global = curb; first = 0; } else { if (jl_binding_kind(curbpart) != rkp->kind || curbpart->restriction != rkp->restriction) return 0; - if ((rkp->kind == BINDING_KIND_GLOBAL || rkp->kind == BINDING_KIND_DECLARED) && rkp->binding_if_global != curb) + if ((rkp->kind == PARTITION_KIND_GLOBAL || rkp->kind == PARTITION_KIND_DECLARED) && rkp->binding_if_global != curb) return 0; } validated_min_world = cur_min_world; @@ -264,10 +264,10 @@ JL_DLLEXPORT int jl_get_binding_leaf_partitions_restriction_kind(jl_binding_t *b } JL_DLLEXPORT jl_value_t *jl_get_binding_leaf_partitions_value_if_const(jl_binding_t *b JL_PROPAGATES_ROOT, int *maybe_depwarn, size_t min_world, size_t max_world) { - struct restriction_kind_pair rkp = { NULL, NULL, BINDING_KIND_GUARD, 0 }; + struct restriction_kind_pair rkp = { NULL, NULL, PARTITION_KIND_GUARD, 0 }; if (!jl_get_binding_leaf_partitions_restriction_kind(b, &rkp, min_world, max_world)) return NULL; - if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != BINDING_KIND_BACKDATED_CONST) { + if (jl_bkind_is_some_constant(rkp.kind) && rkp.kind != PARTITION_KIND_BACKDATED_CONST) { *maybe_depwarn = rkp.maybe_depwarn; return rkp.restriction; } @@ -360,15 +360,15 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( new_bpart = bpart; break; } - } else if (jl_bkind_is_some_import(kind) && kind != BINDING_KIND_IMPLICIT) { + } else if (jl_bkind_is_some_import(kind) && kind != PARTITION_KIND_IMPLICIT) { jl_errorf("cannot declare %s.%s constant; it was already declared as an import", jl_symbol_name(mod->name), jl_symbol_name(var)); - } else if (kind == BINDING_KIND_GLOBAL) { + } else if (kind == PARTITION_KIND_GLOBAL) { jl_errorf("cannot declare %s.%s constant; it was already declared global", jl_symbol_name(mod->name), jl_symbol_name(var)); } if (bpart->min_world == new_world) { - bpart->kind = constant_kind | (bpart->kind & BINDING_FLAG_MASK); + bpart->kind = constant_kind | (bpart->kind & PARTITION_MASK_FLAG); bpart->restriction = val; if (val) jl_gc_wb(bpart, val); @@ -383,7 +383,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( jl_binding_partition_t *prev_bpart = bpart; for (;;) { enum jl_partition_kind prev_kind = jl_binding_kind(prev_bpart); - if (jl_bkind_is_some_constant(prev_kind) || prev_kind == BINDING_KIND_GLOBAL || + if (jl_bkind_is_some_constant(prev_kind) || prev_kind == PARTITION_KIND_GLOBAL || (jl_bkind_is_some_import(prev_kind))) { need_backdate = 0; break; @@ -401,7 +401,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( jl_binding_partition_t *backdate_bpart = new_binding_partition(); new_prev_bpart = backdate_bpart; while (1) { - backdate_bpart->kind = (size_t)BINDING_KIND_BACKDATED_CONST | (prev_bpart->kind & 0xf0); + backdate_bpart->kind = (size_t)PARTITION_KIND_BACKDATED_CONST | (prev_bpart->kind & 0xf0); backdate_bpart->restriction = val; backdate_bpart->min_world = prev_bpart->min_world; jl_gc_wb_fresh(backdate_bpart, val); @@ -549,9 +549,7 @@ static jl_binding_t *new_binding(jl_module_t *mod, jl_sym_t *name) jl_atomic_store_relaxed(&b->partitions, NULL); b->globalref = NULL; b->backedges = NULL; - b->publicp = 0; - b->did_print_backdate_admonition = 0; - b->did_print_implicit_import_admonition = 0; + jl_atomic_store_relaxed(&b->flags, 0); JL_GC_PUSH1(&b); b->globalref = jl_new_globalref(mod, name, b); jl_gc_wb(b, b->globalref); @@ -596,11 +594,11 @@ static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b, jl_module_t *m, jl_sym JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module_t *m, jl_sym_t *s) { jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (jl_options.depwarn && (bpart->kind & BINDING_FLAG_DEPWARN)) { + if (jl_options.depwarn && (bpart->kind & PARTITION_FLAG_DEPWARN)) { jl_binding_deprecation_warning(b); } enum jl_partition_kind kind = jl_binding_kind(bpart); - if (kind != BINDING_KIND_GLOBAL && kind != BINDING_KIND_DECLARED) { + if (kind != PARTITION_KIND_GLOBAL && kind != PARTITION_KIND_DECLARED) { if (jl_bkind_is_some_guard(kind)) { jl_errorf("Global %s.%s does not exist and cannot be assigned.\n" "Note: Julia 1.9 and 1.10 inadvertently omitted this error check (#56933).\n" @@ -642,7 +640,6 @@ JL_DLLEXPORT jl_module_t *jl_get_module_of_binding(jl_module_t *m, jl_sym_t *var static NOINLINE void print_backdate_admonition(jl_binding_t *b) JL_NOTSAFEPOINT { - b->did_print_backdate_admonition = 1; jl_safe_printf( "WARNING: Detected access to binding `%s.%s` in a world prior to its definition world.\n" " Julia 1.12 has introduced more strict world age semantics for global bindings.\n" @@ -654,8 +651,8 @@ static NOINLINE void print_backdate_admonition(jl_binding_t *b) JL_NOTSAFEPOINT static inline void check_backdated_binding(jl_binding_t *b, enum jl_partition_kind kind) JL_NOTSAFEPOINT { - if (__unlikely(kind == BINDING_KIND_BACKDATED_CONST) && - !b->did_print_backdate_admonition) { + if (__unlikely(kind == PARTITION_KIND_BACKDATED_CONST) && + !(jl_atomic_fetch_or(&b->flags, BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION) & BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION)) { print_backdate_admonition(b); } } @@ -791,7 +788,7 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ jl_binding_t *b = jl_get_module_binding(m, var, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); enum jl_partition_kind kind = jl_binding_kind(bpart); - if (kind == BINDING_KIND_GLOBAL || kind == BINDING_KIND_DECLARED || jl_bkind_is_some_constant(kind)) + if (kind == PARTITION_KIND_GLOBAL || kind == PARTITION_KIND_DECLARED || jl_bkind_is_some_constant(kind)) return b; if (jl_bkind_is_some_guard(kind)) { check_safe_newbinding(m, var); @@ -803,7 +800,7 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ if (jl_bkind_is_some_constant(jl_binding_kind(bpart))) f = bpart->restriction; if (f == NULL) { - if (kind == BINDING_KIND_IMPLICIT) { + if (kind == PARTITION_KIND_IMPLICIT) { check_safe_newbinding(m, var); return b; } @@ -814,11 +811,11 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ } int istype = f && jl_is_type(f); if (!istype) { - if (kind == BINDING_KIND_IMPLICIT) { + if (kind == PARTITION_KIND_IMPLICIT) { check_safe_newbinding(m, var); return b; } - else if (kind != BINDING_KIND_IMPORTED) { + else if (kind != PARTITION_KIND_IMPORTED) { // TODO: we might want to require explicitly importing types to add constructors // or we might want to drop this error entirely jl_module_t *from = jl_binding_dbgmodule(b, m, var); @@ -826,15 +823,15 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ jl_module_debug_name(m), jl_module_debug_name(from), jl_symbol_name(var)); } } - else if (kind != BINDING_KIND_IMPORTED) { + else if (kind != PARTITION_KIND_IMPORTED) { int should_error = strcmp(jl_symbol_name(var), "=>") == 0; jl_module_t *from = jl_binding_dbgmodule(b, m, var); if (should_error) { jl_errorf("invalid method definition in %s: function %s.%s must be explicitly imported to be extended", jl_module_debug_name(m), jl_module_debug_name(from), jl_symbol_name(var)); } - else if (!b->did_print_implicit_import_admonition) { - b->did_print_implicit_import_admonition = 1; + else if (jl_atomic_fetch_or(&b->flags, BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION) & + BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION) { jl_printf(JL_STDERR, "WARNING: Constructor for type \"%s\" was extended in `%s` without explicit qualification or import.\n" " NOTE: Assumed \"%s\" refers to `%s.%s`. This behavior is deprecated and may differ in future versions.`\n" " NOTE: This behavior may have differed in Julia versions prior to 1.12.\n" @@ -871,7 +868,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_type(jl_module_t *m, jl_sym_t *var) return jl_nothing; jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); enum jl_partition_kind kind = jl_binding_kind(bpart); - if (jl_bkind_is_some_guard(kind) || kind == BINDING_KIND_DECLARED) + if (jl_bkind_is_some_guard(kind) || kind == PARTITION_KIND_DECLARED) return jl_nothing; if (jl_bkind_is_some_constant(kind)) { // TODO: We would like to return the type of the constant, but @@ -901,7 +898,7 @@ JL_DLLEXPORT int jl_is_imported(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_module_binding(m, var, 0); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - return b && jl_binding_kind(bpart) == BINDING_KIND_IMPORTED; + return b && jl_binding_kind(bpart) == PARTITION_KIND_IMPORTED; } extern const char *jl_filename; @@ -972,7 +969,7 @@ static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl check_safe_import_from(from); jl_binding_t *b = jl_get_binding(from, s); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (bpart->kind & BINDING_FLAG_DEPRECATED) { + if (bpart->kind & PARTITION_FLAG_DEPRECATED) { if (jl_get_binding_value(b) == jl_nothing) { // silently skip importing deprecated values assigned to nothing (to allow later mutation) return; @@ -1013,7 +1010,7 @@ static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl jl_binding_partition_t *btopart = jl_get_binding_partition(bto, new_world); enum jl_partition_kind btokind = jl_binding_kind(btopart); if (jl_bkind_is_some_implicit(btokind)) { - jl_binding_partition_t *new_bpart = jl_replace_binding_locked(bto, btopart, (jl_value_t*)b, (explici != 0) ? BINDING_KIND_IMPORTED : BINDING_KIND_EXPLICIT, new_world); + jl_binding_partition_t *new_bpart = jl_replace_binding_locked(bto, btopart, (jl_value_t*)b, (explici != 0) ? PARTITION_KIND_IMPORTED : PARTITION_KIND_EXPLICIT, new_world); if (jl_atomic_load_relaxed(&new_bpart->max_world) == ~(size_t)0) jl_add_binding_backedge(b, (jl_value_t*)bto); jl_atomic_store_release(&jl_world_counter, new_world); @@ -1021,8 +1018,8 @@ static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl else { if (eq_bindings(bpart, bto, new_world)) { // already imported - potentially upgrade _EXPLICIT to _IMPORTED - if (btokind == BINDING_KIND_EXPLICIT && explici != 0) { - jl_replace_binding_locked(bto, btopart, (jl_value_t*)b, BINDING_KIND_IMPORTED, new_world); + if (btokind == PARTITION_KIND_EXPLICIT && explici != 0) { + jl_replace_binding_locked(bto, btopart, (jl_value_t*)b, PARTITION_KIND_IMPORTED, new_world); jl_atomic_store_release(&jl_world_counter, new_world); } } @@ -1111,14 +1108,14 @@ JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from) if ((void*)b == jl_nothing) break; jl_binding_partition_t *frombpart = jl_get_binding_partition(b, new_world); - if (frombpart->kind & BINDING_FLAG_EXPORTED) { + if (frombpart->kind & PARTITION_FLAG_EXPORTED) { jl_sym_t *var = b->globalref->name; jl_binding_t *tob = jl_get_module_binding(to, var, 0); if (tob) { jl_binding_partition_t *tobpart = jl_get_binding_partition(tob, new_world); enum jl_partition_kind kind = jl_binding_kind(tobpart); if (jl_bkind_is_some_implicit(kind)) { - jl_replace_binding_locked(tob, tobpart, NULL, BINDING_KIND_IMPLICIT_RECOMPUTE, new_world); + jl_replace_binding_locked(tob, tobpart, NULL, PARTITION_KIND_IMPLICIT_RECOMPUTE, new_world); } } } @@ -1152,8 +1149,8 @@ JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported) JL_LOCK(&world_counter_lock); size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); - int was_exported = (bpart->kind & BINDING_FLAG_EXPORTED) != 0; - if (b->publicp) { + int was_exported = (bpart->kind & PARTITION_FLAG_EXPORTED) != 0; + if (jl_atomic_load_relaxed(&b->flags) & BINDING_FLAG_PUBLICP) { // check for conflicting declarations if (was_exported && !exported) jl_errorf("cannot declare %s.%s public; it is already declared exported", @@ -1162,9 +1159,9 @@ JL_DLLEXPORT void jl_module_public(jl_module_t *from, jl_sym_t *s, int exported) jl_errorf("cannot declare %s.%s exported; it is already declared public", jl_symbol_name(from->name), jl_symbol_name(s)); } - b->publicp = 1; + jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_PUBLICP); if (was_exported != exported) { - jl_replace_binding_locked2(b, bpart, bpart->restriction, bpart->kind | BINDING_FLAG_EXPORTED, new_world); + jl_replace_binding_locked2(b, bpart, bpart->restriction, bpart->kind | PARTITION_FLAG_EXPORTED, new_world); jl_atomic_store_release(&jl_world_counter, new_world); } JL_UNLOCK(&world_counter_lock); @@ -1197,20 +1194,20 @@ JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_module_binding(m, var, 0); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - return b && ((bpart->kind & BINDING_FLAG_EXPORTED) || jl_binding_kind(bpart) == BINDING_KIND_GLOBAL); + return b && ((bpart->kind & PARTITION_FLAG_EXPORTED) || jl_binding_kind(bpart) == PARTITION_KIND_GLOBAL); } JL_DLLEXPORT int jl_module_exports_p(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_module_binding(m, var, 0); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - return b && (bpart->kind & BINDING_FLAG_EXPORTED); + return b && (bpart->kind & PARTITION_FLAG_EXPORTED); } JL_DLLEXPORT int jl_module_public_p(jl_module_t *m, jl_sym_t *var) { jl_binding_t *b = jl_get_module_binding(m, var, 0); - return b && b->publicp; + return b && (jl_atomic_load_relaxed(&b->flags) & BINDING_FLAG_PUBLICP); } uint_t bindingkey_hash(size_t idx, jl_value_t *data) @@ -1305,7 +1302,7 @@ JL_DLLEXPORT void jl_set_const(jl_module_t *m JL_ROOTING_ARGUMENT, jl_sym_t *var jl_binding_partition_t *bpart = jl_get_binding_partition(bp, jl_current_task->world_age); bpart->min_world = 0; jl_atomic_store_release(&bpart->max_world, ~(size_t)0); - bpart->kind = BINDING_KIND_CONST | (bpart->kind & BINDING_FLAG_MASK); + bpart->kind = PARTITION_KIND_CONST | (bpart->kind & PARTITION_MASK_FLAG); bpart->restriction = val; jl_gc_wb(bpart, val); } @@ -1364,7 +1361,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked(jl_binding_t *b, jl_binding_partition_t *old_bpart, jl_value_t *restriction_val, enum jl_partition_kind kind, size_t new_world) { // Copy flags from old bpart - return jl_replace_binding_locked2(b, old_bpart, restriction_val, (size_t)kind | (size_t)(old_bpart->kind & BINDING_FLAG_MASK), + return jl_replace_binding_locked2(b, old_bpart, restriction_val, (size_t)kind | (size_t)(old_bpart->kind & PARTITION_MASK_FLAG), new_world); } @@ -1386,10 +1383,10 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, jl_binding_partition_t *new_bpart = new_binding_partition(); JL_GC_PUSH1(&new_bpart); new_bpart->min_world = new_world; - if ((kind & BINDING_KIND_MASK) == BINDING_KIND_IMPLICIT_RECOMPUTE) { + if ((kind & PARTITION_MASK_KIND) == PARTITION_KIND_IMPLICIT_RECOMPUTE) { assert(!restriction_val); jl_check_new_binding_implicit(new_bpart /* callee rooted */, b, NULL, new_world); - new_bpart->kind |= kind & BINDING_FLAG_MASK; + new_bpart->kind |= kind & PARTITION_MASK_FLAG; } else { new_bpart->kind = kind; @@ -1399,7 +1396,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, jl_atomic_store_relaxed(&new_bpart->next, old_bpart); jl_gc_wb_fresh(new_bpart, old_bpart); - if (((old_bpart->kind & BINDING_FLAG_EXPORTED) || (kind & BINDING_FLAG_EXPORTED)) && jl_require_world != ~(size_t)0) { + if (((old_bpart->kind & PARTITION_FLAG_EXPORTED) || (kind & PARTITION_FLAG_EXPORTED)) && jl_require_world != ~(size_t)0) { jl_atomic_store_release(&b->globalref->mod->export_set_changed_since_require_world, 1); } @@ -1454,13 +1451,13 @@ JL_DLLEXPORT void jl_disable_binding(jl_globalref_t *gr) b = jl_get_module_binding(gr->mod, gr->name, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (jl_binding_kind(bpart) == BINDING_KIND_GUARD) { + if (jl_binding_kind(bpart) == PARTITION_KIND_GUARD) { // Already guard return; } for (;;) - if (jl_replace_binding(b, bpart, NULL, BINDING_KIND_GUARD)) + if (jl_replace_binding(b, bpart, NULL, PARTITION_KIND_GUARD)) break; } @@ -1474,12 +1471,12 @@ JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var) // set the deprecated flag for a binding: // 0=not deprecated, 1=renamed, 2=moved to another package -static const size_t DEPWARN_FLAGS = BINDING_FLAG_DEPRECATED | BINDING_FLAG_DEPWARN; +static const size_t DEPWARN_FLAGS = PARTITION_FLAG_DEPRECATED | PARTITION_FLAG_DEPWARN; JL_DLLEXPORT void jl_deprecate_binding(jl_module_t *m, jl_sym_t *var, int flag) { jl_binding_t *b = jl_get_binding(m, var); - size_t new_flags = flag == 1 ? BINDING_FLAG_DEPRECATED | BINDING_FLAG_DEPWARN : - flag == 2 ? BINDING_FLAG_DEPRECATED : + size_t new_flags = flag == 1 ? PARTITION_FLAG_DEPRECATED | PARTITION_FLAG_DEPWARN : + flag == 2 ? PARTITION_FLAG_DEPRECATED : 0; JL_LOCK(&world_counter_lock); size_t new_world = jl_atomic_load_acquire(&jl_world_counter)+1; @@ -1507,9 +1504,9 @@ static int should_depwarn(jl_binding_t *b, uint8_t flag) // an appropriate warning). for (;;) { jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (bpart->kind & BINDING_FLAG_DEPRECATED) + if (bpart->kind & PARTITION_FLAG_DEPRECATED) return 1; - if ((bpart->kind & BINDING_KIND_MASK) != BINDING_KIND_IMPLICIT) + if ((bpart->kind & PARTITION_MASK_KIND) != PARTITION_KIND_IMPLICIT) break; b = (jl_binding_t*)bpart->restriction; } @@ -1518,7 +1515,7 @@ static int should_depwarn(jl_binding_t *b, uint8_t flag) JL_DLLEXPORT void jl_binding_deprecation_check(jl_binding_t *b) { - if (jl_options.depwarn && should_depwarn(b, BINDING_FLAG_DEPWARN)) + if (jl_options.depwarn && should_depwarn(b, PARTITION_FLAG_DEPWARN)) jl_binding_deprecation_warning(b); } @@ -1527,7 +1524,7 @@ JL_DLLEXPORT int jl_is_binding_deprecated(jl_module_t *m, jl_sym_t *var) jl_binding_t *b = jl_get_module_binding(m, var, 0); if (!b) return 0; - return should_depwarn(b, BINDING_FLAG_DEPRECATED); + return should_depwarn(b, PARTITION_FLAG_DEPRECATED); } void jl_binding_deprecation_warning(jl_binding_t *b) @@ -1538,13 +1535,13 @@ void jl_binding_deprecation_warning(jl_binding_t *b) jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); int first = 0; - while (!(bpart->kind & BINDING_FLAG_DEPWARN)) { + while (!(bpart->kind & PARTITION_FLAG_DEPWARN)) { if (first) { jl_printf(JL_STDERR, "binding implicitly imported via "); first = 0; } jl_printf(JL_STDERR, "%s.%s -> ", jl_symbol_name(b->globalref->mod->name), jl_symbol_name(b->globalref->name)); - assert(jl_binding_kind(bpart) == BINDING_KIND_IMPLICIT); + assert(jl_binding_kind(bpart) == PARTITION_KIND_IMPLICIT); b = (jl_binding_t*)bpart->restriction; bpart = jl_get_binding_partition(b, jl_current_task->world_age); } @@ -1572,8 +1569,8 @@ jl_value_t *jl_check_binding_assign_value(jl_binding_t *b JL_PROPAGATES_ROOT, jl JL_GC_PUSH1(&rhs); // callee-rooted jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); enum jl_partition_kind kind = jl_binding_kind(bpart); - assert(kind == BINDING_KIND_DECLARED || kind == BINDING_KIND_GLOBAL); - jl_value_t *old_ty = kind == BINDING_KIND_DECLARED ? (jl_value_t*)jl_any_type : bpart->restriction; + assert(kind == PARTITION_KIND_DECLARED || kind == PARTITION_KIND_GLOBAL); + jl_value_t *old_ty = kind == PARTITION_KIND_DECLARED ? (jl_value_t*)jl_any_type : bpart->restriction; JL_GC_PROMISE_ROOTED(old_ty); if (old_ty != (jl_value_t*)jl_any_type && jl_typeof(rhs) != old_ty) { if (!jl_isa(rhs, old_ty)) @@ -1664,11 +1661,11 @@ void append_module_names(jl_array_t* a, jl_module_t *m, int all, int imported, i int main_public = (m == jl_main_module && !(asname == jl_eval_sym || asname == jl_include_sym)); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); enum jl_partition_kind kind = jl_binding_kind(bpart); - if (((b->publicp) || - (imported && (kind == BINDING_KIND_CONST_IMPORT || kind == BINDING_KIND_IMPORTED)) || - (usings && kind == BINDING_KIND_EXPLICIT) || - ((kind == BINDING_KIND_GLOBAL || kind == BINDING_KIND_CONST || kind == BINDING_KIND_DECLARED) && (all || main_public))) && - (all || (!(bpart->kind & BINDING_FLAG_DEPRECATED) && !hidden))) + if (((jl_atomic_load_relaxed(&b->flags) & BINDING_FLAG_PUBLICP) || + (imported && (kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_IMPORTED)) || + (usings && kind == PARTITION_KIND_EXPLICIT) || + ((kind == PARTITION_KIND_GLOBAL || kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_DECLARED) && (all || main_public))) && + (all || (!(bpart->kind & PARTITION_FLAG_DEPRECATED) && !hidden))) _append_symbol_to_bindings_array(a, asname); } } @@ -1681,7 +1678,7 @@ void append_exported_names(jl_array_t* a, jl_module_t *m, int all) if ((void*)b == jl_nothing) break; jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if ((bpart->kind & BINDING_FLAG_EXPORTED) && (all || !(bpart->kind & BINDING_FLAG_DEPRECATED))) + if ((bpart->kind & PARTITION_FLAG_EXPORTED) && (all || !(bpart->kind & PARTITION_FLAG_DEPRECATED))) _append_symbol_to_bindings_array(a, b->globalref->name); } } @@ -1751,7 +1748,7 @@ JL_DLLEXPORT void jl_clear_implicit_imports(jl_module_t *m) if ((void*)b == jl_nothing) break; jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (jl_binding_kind(bpart) == BINDING_KIND_IMPLICIT) { + if (jl_binding_kind(bpart) == PARTITION_KIND_IMPLICIT) { jl_atomic_store_relaxed(&b->partitions, NULL); } } diff --git a/src/staticdata.c b/src/staticdata.c index fa6b709160804..d9c2912673708 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3553,10 +3553,10 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t if (jl_atomic_load_relaxed(&bpart->max_world) != ~(size_t)0) return 1; size_t raw_kind = bpart->kind; - enum jl_partition_kind kind = (enum jl_partition_kind)(raw_kind & BINDING_KIND_MASK); + enum jl_partition_kind kind = (enum jl_partition_kind)(raw_kind & PARTITION_MASK_KIND); if (!unchanged_implicit && jl_bkind_is_some_implicit(kind)) { jl_check_new_binding_implicit(bpart, b, NULL, jl_atomic_load_relaxed(&jl_world_counter)); - bpart->kind |= (raw_kind & BINDING_FLAG_MASK); + bpart->kind |= (raw_kind & PARTITION_MASK_FLAG); if (bpart->min_world > jl_require_world) goto invalidated; } @@ -3571,7 +3571,7 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t if (latest_imported_bpart->min_world <= bpart->min_world) { add_backedge: // Imported binding is still valid - if ((kind == BINDING_KIND_EXPLICIT || kind == BINDING_KIND_IMPORTED) && + if ((kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) && external_blob_index((jl_value_t*)imported_binding) != mod_idx) { jl_add_binding_backedge(imported_binding, (jl_value_t*)b); } @@ -3596,7 +3596,7 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t jl_validate_binding_partition(bedge, jl_atomic_load_relaxed(&bedge->partitions), mod_idx, 0, 0); } } - if (bpart->kind & BINDING_FLAG_EXPORTED) { + if (bpart->kind & PARTITION_FLAG_EXPORTED) { jl_module_t *mod = b->globalref->mod; jl_sym_t *name = b->globalref->name; JL_LOCK(&mod->lock); diff --git a/src/toplevel.c b/src/toplevel.c index a25a3794144ae..35ec0684ab7a7 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -303,15 +303,15 @@ void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type, in jl_binding_partition_t *bpart = NULL; if (!strong && set_type) jl_error("Weak global definitions cannot have types"); - enum jl_partition_kind new_kind = strong ? BINDING_KIND_GLOBAL : BINDING_KIND_DECLARED; + enum jl_partition_kind new_kind = strong ? PARTITION_KIND_GLOBAL : PARTITION_KIND_DECLARED; jl_value_t *global_type = set_type; if (strong && !global_type) global_type = (jl_value_t*)jl_any_type; while (1) { bpart = jl_get_binding_partition(b, new_world); enum jl_partition_kind kind = jl_binding_kind(bpart); - if (kind != BINDING_KIND_GLOBAL) { - if (jl_bkind_is_some_guard(kind) || kind == BINDING_KIND_DECLARED || kind == BINDING_KIND_IMPLICIT) { + if (kind != PARTITION_KIND_GLOBAL) { + if (jl_bkind_is_some_guard(kind) || kind == PARTITION_KIND_DECLARED || kind == PARTITION_KIND_IMPLICIT) { if (kind == new_kind) { if (!set_type) goto done; @@ -319,7 +319,7 @@ void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type, in } check_safe_newbinding(gm, gs); if (bpart->min_world == new_world) { - bpart->kind = new_kind | (bpart->kind & BINDING_FLAG_MASK); + bpart->kind = new_kind | (bpart->kind & PARTITION_MASK_FLAG); bpart->restriction = global_type; if (global_type) jl_gc_wb(bpart, global_type); @@ -659,10 +659,10 @@ static void import_module(jl_task_t *ct, jl_module_t *JL_NONNULL m, jl_module_t jl_binding_t *b = jl_get_module_binding(m, name, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, ct->world_age); enum jl_partition_kind kind = jl_binding_kind(bpart); - if (kind != BINDING_KIND_GUARD && kind != BINDING_KIND_FAILED && kind != BINDING_KIND_DECLARED && kind != BINDING_KIND_IMPLICIT) { + if (kind != PARTITION_KIND_GUARD && kind != PARTITION_KIND_FAILED && kind != PARTITION_KIND_DECLARED && kind != PARTITION_KIND_IMPLICIT) { // Unlike regular constant declaration, we allow this as long as we eventually end up at a constant. jl_walk_binding_inplace(&b, &bpart, ct->world_age); - if (jl_binding_kind(bpart) == BINDING_KIND_CONST || jl_binding_kind(bpart) == BINDING_KIND_BACKDATED_CONST || jl_binding_kind(bpart) == BINDING_KIND_CONST_IMPORT) { + if (jl_binding_kind(bpart) == PARTITION_KIND_CONST || jl_binding_kind(bpart) == PARTITION_KIND_BACKDATED_CONST || jl_binding_kind(bpart) == PARTITION_KIND_CONST_IMPORT) { // Already declared (e.g. on another thread) or imported. if (bpart->restriction == (jl_value_t*)import) return; @@ -670,7 +670,7 @@ static void import_module(jl_task_t *ct, jl_module_t *JL_NONNULL m, jl_module_t jl_errorf("importing %s into %s conflicts with an existing global", jl_symbol_name(name), jl_symbol_name(m->name)); } - jl_declare_constant_val2(b, m, name, (jl_value_t*)import, BINDING_KIND_CONST_IMPORT); + jl_declare_constant_val2(b, m, name, (jl_value_t*)import, PARTITION_KIND_CONST_IMPORT); } // in `import A.B: x, y, ...`, evaluate the `A.B` part if it exists @@ -747,7 +747,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val2( JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val(jl_binding_t *b, jl_module_t *mod, jl_sym_t *var, jl_value_t *val) { - return jl_declare_constant_val2(b, mod, var, val, val ? BINDING_KIND_CONST : BINDING_KIND_UNDEF_CONST); + return jl_declare_constant_val2(b, mod, var, val, val ? PARTITION_KIND_CONST : PARTITION_KIND_UNDEF_CONST); } JL_DLLEXPORT void jl_eval_const_decl(jl_module_t *m, jl_value_t *arg, jl_value_t *val) diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index bbc2c9a6d44b0..99142d8a64558 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -35,14 +35,14 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) if scope isa Module bpart = Base.lookup_binding_partition(ex.world, GlobalRef(scope, var)) kind = Base.binding_kind(bpart) - if kind === Base.BINDING_KIND_GLOBAL || kind === Base.BINDING_KIND_UNDEF_CONST || kind == Base.BINDING_KIND_DECLARED + if kind === Base.PARTITION_KIND_GLOBAL || kind === Base.PARTITION_KIND_UNDEF_CONST || kind == Base.PARTITION_KIND_DECLARED print(io, "\nSuggestion: add an appropriate import or assignment. This global was declared but not assigned.") - elseif kind === Base.BINDING_KIND_FAILED + elseif kind === Base.PARTITION_KIND_FAILED print(io, "\nHint: It looks like two or more modules export different ", "bindings with this name, resulting in ambiguity. Try explicitly ", "importing it from a particular module, or qualifying the name ", "with the module it should come from.") - elseif kind === Base.BINDING_KIND_GUARD + elseif kind === Base.PARTITION_KIND_GUARD print(io, "\nSuggestion: check for spelling errors or missing imports.") elseif Base.is_some_imported(kind) print(io, "\nSuggestion: this global was defined as `$(Base.partition_restriction(bpart).globalref)` but not assigned a value.") diff --git a/stdlib/REPL/src/REPLCompletions.jl b/stdlib/REPL/src/REPLCompletions.jl index a0a0d4b1fa8d7..d5a920efafe4e 100644 --- a/stdlib/REPL/src/REPLCompletions.jl +++ b/stdlib/REPL/src/REPLCompletions.jl @@ -648,7 +648,7 @@ function CC.abstract_eval_globalref(interp::REPLInterpreter, g::GlobalRef, baile return CC.RTEffects(Const(CC.partition_restriction(partition)), Union{}, CC.EFFECTS_TOTAL) else b = convert(Core.Binding, g) - if CC.binding_kind(partition) == CC.BINDING_KIND_GLOBAL && isdefined(b, :value) + if CC.binding_kind(partition) == CC.PARTITION_KIND_GLOBAL && isdefined(b, :value) return CC.RTEffects(Const(b.value), Union{}, CC.EFFECTS_TOTAL) end end diff --git a/stdlib/REPL/src/docview.jl b/stdlib/REPL/src/docview.jl index 6a73a9631a49a..5e0a30abb4137 100644 --- a/stdlib/REPL/src/docview.jl +++ b/stdlib/REPL/src/docview.jl @@ -313,7 +313,7 @@ function summarize(binding::Binding, sig) println(io, "No documentation found.\n") quot = any(isspace, sprint(print, binding)) ? "'" : "" bpart = Base.lookup_binding_partition(Base.tls_world_age(), convert(Core.Binding, GlobalRef(binding.mod, binding.var))) - if Base.binding_kind(bpart) === Base.BINDING_KIND_GUARD + if Base.binding_kind(bpart) === Base.PARTITION_KIND_GUARD println(io, "Binding ", quot, "`", binding, "`", quot, " does not exist.") else println(io, "Binding ", quot, "`", binding, "`", quot, " exists, but has not been assigned a value.") diff --git a/test/rebinding.jl b/test/rebinding.jl index b78d933ae9101..8f54aa99b15bd 100644 --- a/test/rebinding.jl +++ b/test/rebinding.jl @@ -4,24 +4,24 @@ module Rebinding using Test make_foo() = Foo(1) - @test Base.binding_kind(@__MODULE__, :Foo) == Base.BINDING_KIND_GUARD + @test Base.binding_kind(@__MODULE__, :Foo) == Base.PARTITION_KIND_GUARD struct Foo x::Int end const defined_world_age = Base.tls_world_age() x = Foo(1) - @test Base.binding_kind(@__MODULE__, :Foo) == Base.BINDING_KIND_CONST + @test Base.binding_kind(@__MODULE__, :Foo) == Base.PARTITION_KIND_CONST @test !contains(repr(x), "@world") Base.delete_binding(@__MODULE__, :Foo) - @test Base.binding_kind(@__MODULE__, :Foo) == Base.BINDING_KIND_GUARD + @test Base.binding_kind(@__MODULE__, :Foo) == Base.PARTITION_KIND_GUARD @test contains(repr(x), "@world") # Test that it still works if Foo is redefined to a non-type const Foo = 1 - @test Base.binding_kind(@__MODULE__, :Foo) == Base.BINDING_KIND_CONST + @test Base.binding_kind(@__MODULE__, :Foo) == Base.PARTITION_KIND_CONST @test contains(repr(x), "@world") Base.delete_binding(@__MODULE__, :Foo) From 686c91793bb3e6bdfe34f93f070a35d03c4e80a1 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 3 Mar 2025 23:23:11 -0500 Subject: [PATCH 27/74] Align module base between invalidation and edge tracking (#57625) Our implicit edge tracking for bindings does not explicitly store any edges for bindings in the *current* module. The idea behind this is that this is a good time-space tradeoff for validation, because substantially all binding references in a module will be to its defining module, while the total number of methods within a module is limited and substantially smaller than the total number of methods in the entire system. However, we have an issue where the code that stores these edges and the invalidation code disagree on which module is the *current* one. The edge storing code was using the module in which the method was defined, while the invalidation code was using the one in which the MethodTable is defined. With these being misaligned, we can miss necessary invalidations. Both options are in principle possible, but I think the former is better, because the module in which the method is defined is also the module that we are likely to have a lot of references to (since they get referenced implicitly by just writing symbols in the code). However, this presents a problem: We don't actually have a way to iterate all the methods defined in a particular module, without just doing the brute force thing of scanning all methods and filtering. To address this, build on the deferred scanning code added in #57615 to also add any scanned modules to an explicit list in `Module`. This costs some space, but only proportional to the number of defined methods, (and thus proportional to the written source code). Note that we don't actually observe any issues in the test suite on master due to this bug. However, this is because we are grossly over-invalidating, which hides the missing invalidations from this issue (#57617). (cherry picked from commit 274d80e3bdc1446ce4855fca41e68b03c273e390) --- Compiler/src/abstractinterpretation.jl | 18 +++--- Compiler/src/utilities.jl | 19 ++++++ base/expr.jl | 4 ++ base/invalidation.jl | 59 +++++++++++------- base/runtime_internals.jl | 2 + src/gc-stock.c | 3 + src/gf.c | 49 +++++++++++---- src/jltypes.c | 8 ++- src/julia.h | 11 +++- src/julia_internal.h | 3 +- src/method.c | 45 ++++++++++++-- src/module.c | 83 +++++++++++++++++++++----- src/staticdata.c | 4 ++ test/rebinding.jl | 20 +++++++ 14 files changed, 264 insertions(+), 64 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 65443bef73c51..212169c107942 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3594,18 +3594,20 @@ scan_partitions(query::Function, interp, g::GlobalRef, wwr::WorldWithRange) = abstract_load_all_consistent_leaf_partitions(interp, g::GlobalRef, wwr::WorldWithRange) = scan_leaf_partitions(abstract_eval_partition_load, interp, g, wwr) +function abstract_eval_globalref_partition(interp, binding::Core.Binding, partition::Core.BindingPartition) + # For inference purposes, we don't particularly care which global binding we end up loading, we only + # care about its type. However, we would still like to terminate the world range for the particular + # binding we end up reaching such that codegen can emit a simpler pointer load. + Pair{RTEffects, Union{Nothing, Core.Binding}}( + abstract_eval_partition_load(interp, partition), + binding_kind(partition) in (PARTITION_KIND_GLOBAL, PARTITION_KIND_DECLARED) ? binding : nothing) +end + function abstract_eval_globalref(interp, g::GlobalRef, saw_latestworld::Bool, sv::AbsIntState) if saw_latestworld return RTEffects(Any, Any, generic_getglobal_effects) end - (valid_worlds, (ret, binding_if_global)) = scan_leaf_partitions(interp, g, sv.world) do interp, binding, partition - # For inference purposes, we don't particularly care which global binding we end up loading, we only - # care about its type. However, we would still like to terminate the world range for the particular - # binding we end up reaching such that codegen can emit a simpler pointer load. - Pair{RTEffects, Union{Nothing, Core.Binding}}( - abstract_eval_partition_load(interp, partition), - binding_kind(partition) in (PARTITION_KIND_GLOBAL, PARTITION_KIND_DECLARED) ? binding : nothing) - end + (valid_worlds, (ret, binding_if_global)) = scan_leaf_partitions(abstract_eval_globalref_partition, interp, g, sv.world) update_valid_age!(sv, valid_worlds) if ret.rt !== Union{} && ret.exct === UndefVarError && binding_if_global !== nothing && InferenceParams(interp).assume_bindings_static if isdefined(binding_if_global, :value) diff --git a/Compiler/src/utilities.jl b/Compiler/src/utilities.jl index c322d1062cea1..6113bdb14e7df 100644 --- a/Compiler/src/utilities.jl +++ b/Compiler/src/utilities.jl @@ -129,6 +129,25 @@ function retrieve_code_info(mi::MethodInstance, world::UInt) else c = copy(src::CodeInfo) end + if (def.did_scan_source & 0x1) == 0x0 + # This scan must happen: + # 1. After method definition + # 2. Before any code instances that may have relied on information + # from implicit GlobalRefs for this method are added to the cache + # 3. Preferably while the IR is already uncompressed + # 4. As late as possible, as early adding of the backedges may cause + # spurious invalidations. + # + # At the moment we do so here, because + # 1. It's reasonably late + # 2. It has easy access to the uncompressed IR + # 3. We necessarily pass through here before relying on any + # information obtained from implicit GlobalRefs. + # + # However, the exact placement of this scan is not as important as + # long as the above conditions are met. + ccall(:jl_scan_method_source_now, Cvoid, (Any, Any), def, c) + end end if c isa CodeInfo c.parent = mi diff --git a/base/expr.jl b/base/expr.jl index 8ae394e122443..dd89f9e64abd4 100644 --- a/base/expr.jl +++ b/base/expr.jl @@ -1344,6 +1344,10 @@ function make_atomic(order, ex) op = :+ elseif ex.head === :(-=) op = :- + elseif ex.head === :(|=) + op = :| + elseif ex.head === :(&=) + op = :& elseif @isdefined string shead = string(ex.head) if endswith(shead, '=') diff --git a/base/invalidation.jl b/base/invalidation.jl index 539999c15dd1d..5b48af0171205 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -113,15 +113,34 @@ function invalidate_method_for_globalref!(gr::GlobalRef, method::Method, invalid end end -function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core.BindingPartition, new_bpart::Union{Core.BindingPartition, Nothing}, new_max_world::UInt) +export_affecting_partition_flags(bpart::Core.BindingPartition) = + ((bpart.kind & PARTITION_MASK_KIND) == PARTITION_KIND_GUARD, + (bpart.kind & PARTITION_FLAG_EXPORTED) != 0, + (bpart.kind & PARTITION_FLAG_DEPRECATED) != 0) + +function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) gr = b.globalref - if !is_some_guard(binding_kind(invalidated_bpart)) - # TODO: We may want to invalidate for these anyway, since they have performance implications - foreach_module_mtable(gr.mod, new_max_world) do mt::Core.MethodTable - for method in MethodList(mt) + + (_, (ib, ibpart)) = Compiler.walk_binding_partition(b, invalidated_bpart, new_max_world) + (_, (nb, nbpart)) = Compiler.walk_binding_partition(b, new_bpart, new_max_world+1) + + # abstract_eval_globalref_partition is the maximum amount of information that inference + # reads from a binding partition. If this information does not change - we do not need to + # invalidate any code that inference created, because we know that the result will not change. + need_to_invalidate_code = + Compiler.abstract_eval_globalref_partition(nothing, ib, ibpart) !== + Compiler.abstract_eval_globalref_partition(nothing, nb, nbpart) + + need_to_invalidate_export = export_affecting_partition_flags(invalidated_bpart) !== + export_affecting_partition_flags(new_bpart) + + if need_to_invalidate_code + if (b.flags & BINDING_FLAG_ANY_IMPLICIT_EDGES) != 0 + nmethods = ccall(:jl_module_scanned_methods_length, Csize_t, (Any,), gr.mod) + for i = 1:nmethods + method = ccall(:jl_module_scanned_methods_getindex, Any, (Any, Csize_t), gr.mod, i)::Method invalidate_method_for_globalref!(gr, method, invalidated_bpart, new_max_world) end - return true end if isdefined(b, :backedges) for edge in b.backedges @@ -133,27 +152,32 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core latest_bpart.max_world == typemax(UInt) || continue is_some_imported(binding_kind(latest_bpart)) || continue partition_restriction(latest_bpart) === b || continue - invalidate_code_for_globalref!(edge, latest_bpart, nothing, new_max_world) + invalidate_code_for_globalref!(edge, latest_bpart, latest_bpart, new_max_world) else invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) end end end end - if (invalidated_bpart.kind & PARTITION_FLAG_EXPORTED != 0) || (new_bpart !== nothing && (new_bpart.kind & PARTITION_FLAG_EXPORTED != 0)) + + if need_to_invalidate_code || need_to_invalidate_export # This binding was exported - we need to check all modules that `using` us to see if they # have a binding that is affected by this change. usings_backedges = ccall(:jl_get_module_usings_backedges, Any, (Any,), gr.mod) if usings_backedges !== nothing - for user in usings_backedges::Vector{Any} + for user::Module in usings_backedges::Vector{Any} user_binding = ccall(:jl_get_module_binding_or_nothing, Any, (Any, Any), user, gr.name) user_binding === nothing && continue isdefined(user_binding, :partitions) || continue latest_bpart = user_binding.partitions latest_bpart.max_world == typemax(UInt) || continue binding_kind(latest_bpart) in (PARTITION_KIND_IMPLICIT, PARTITION_KIND_FAILED, PARTITION_KIND_GUARD) || continue - @atomic :release latest_bpart.max_world = new_max_world - invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, nothing, new_max_world) + new_bpart = need_to_invalidate_export ? + ccall(:jl_maybe_reresolve_implicit, Any, (Any, Any, Csize_t), user_binding, latest_bpart, new_max_world) : + latest_bpart + if need_to_invalidate_code || new_bpart !== latest_bpart + invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, new_bpart, new_max_world) + end end end end @@ -161,17 +185,10 @@ end invalidate_code_for_globalref!(gr::GlobalRef, invalidated_bpart::Core.BindingPartition, new_bpart::Core.BindingPartition, new_max_world::UInt) = invalidate_code_for_globalref!(convert(Core.Binding, gr), invalidated_bpart, new_bpart, new_max_world) -gr_needs_backedge_in_module(gr::GlobalRef, mod::Module) = gr.mod !== mod - -# N.B.: This needs to match jl_maybe_add_binding_backedge function maybe_add_binding_backedge!(b::Core.Binding, edge::Union{Method, CodeInstance}) - method = isa(edge, Method) ? edge : edge.def.def::Method - gr_needs_backedge_in_module(b.globalref, method.module) || return - if !isdefined(b, :backedges) - b.backedges = Any[] - end - !isempty(b.backedges) && b.backedges[end] === edge && return - push!(b.backedges, edge) + meth = isa(edge, Method) ? edge : Compiler.get_ci_mi(edge).def + ccall(:jl_maybe_add_binding_backedge, Cint, (Any, Any, Any), b, edge, meth) + return nothing end function binding_was_invalidated(b::Core.Binding) diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 536705f1bdc2a..22fba966679d9 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -216,6 +216,8 @@ const PARTITION_FLAG_DEPWARN = 0x40 const PARTITION_MASK_KIND = 0x0f const PARTITION_MASK_FLAG = 0xf0 +const BINDING_FLAG_ANY_IMPLICIT_EDGES = 0x8 + is_defined_const_binding(kind::UInt8) = (kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_BACKDATED_CONST) is_some_const_binding(kind::UInt8) = (is_defined_const_binding(kind) || kind == PARTITION_KIND_UNDEF_CONST) is_some_imported(kind::UInt8) = (kind == PARTITION_KIND_IMPLICIT || kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) diff --git a/src/gc-stock.c b/src/gc-stock.c index c9cb57b19a604..3b49b82caf530 100644 --- a/src/gc-stock.c +++ b/src/gc-stock.c @@ -2147,6 +2147,9 @@ STATIC_INLINE void gc_mark_module_binding(jl_ptls_t ptls, jl_module_t *mb_parent gc_assert_parent_validity((jl_value_t *)mb_parent, (jl_value_t *)mb_parent->usings_backedges); gc_try_claim_and_push(mq, (jl_value_t *)mb_parent->usings_backedges, &nptr); gc_heap_snapshot_record_binding_partition_edge((jl_value_t*)mb_parent, mb_parent->usings_backedges); + gc_assert_parent_validity((jl_value_t *)mb_parent, (jl_value_t *)mb_parent->scanned_methods); + gc_try_claim_and_push(mq, (jl_value_t *)mb_parent->scanned_methods, &nptr); + gc_heap_snapshot_record_binding_partition_edge((jl_value_t*)mb_parent, mb_parent->scanned_methods); size_t nusings = module_usings_length(mb_parent); if (nusings > 0) { // this is only necessary because bindings for "using" modules diff --git a/src/gf.c b/src/gf.c index 1d93d03cc59d2..c07b90a5c12a2 100644 --- a/src/gf.c +++ b/src/gf.c @@ -1839,7 +1839,7 @@ JL_DLLEXPORT jl_value_t *jl_debug_method_invalidation(int state) return jl_nothing; } -static void _invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_world, int depth); +static void _invalidate_backedges(jl_method_instance_t *replaced_mi, jl_code_instance_t *replaced_ci, size_t max_world, int depth); // recursively invalidate cached methods that had an edge to a replaced method static void invalidate_code_instance(jl_code_instance_t *replaced, size_t max_world, int depth) @@ -1858,13 +1858,15 @@ static void invalidate_code_instance(jl_code_instance_t *replaced, size_t max_wo if (!jl_is_method(replaced_mi->def.method)) return; // shouldn't happen, but better to be safe JL_LOCK(&replaced_mi->def.method->writelock); - if (jl_atomic_load_relaxed(&replaced->max_world) == ~(size_t)0) { + size_t replacedmaxworld = jl_atomic_load_relaxed(&replaced->max_world); + if (replacedmaxworld == ~(size_t)0) { assert(jl_atomic_load_relaxed(&replaced->min_world) - 1 <= max_world && "attempting to set illogical world constraints (probable race condition)"); jl_atomic_store_release(&replaced->max_world, max_world); + // recurse to all backedges to update their valid range also + _invalidate_backedges(replaced_mi, replaced, max_world, depth + 1); + } else { + assert(jl_atomic_load_relaxed(&replaced->max_world) <= max_world); } - assert(jl_atomic_load_relaxed(&replaced->max_world) <= max_world); - // recurse to all backedges to update their valid range also - _invalidate_backedges(replaced_mi, max_world, depth + 1); JL_UNLOCK(&replaced_mi->def.method->writelock); } @@ -1873,19 +1875,42 @@ JL_DLLEXPORT void jl_invalidate_code_instance(jl_code_instance_t *replaced, size invalidate_code_instance(replaced, max_world, 1); } -static void _invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_world, int depth) { +static void _invalidate_backedges(jl_method_instance_t *replaced_mi, jl_code_instance_t *replaced_ci, size_t max_world, int depth) { jl_array_t *backedges = replaced_mi->backedges; if (backedges) { // invalidate callers (if any) replaced_mi->backedges = NULL; JL_GC_PUSH1(&backedges); size_t i = 0, l = jl_array_nrows(backedges); + size_t ins = 0; jl_code_instance_t *replaced; while (i < l) { - i = get_next_edge(backedges, i, NULL, &replaced); + jl_value_t *invokesig = NULL; + i = get_next_edge(backedges, i, &invokesig, &replaced); JL_GC_PROMISE_ROOTED(replaced); // propagated by get_next_edge from backedges + if (replaced_ci) { + // If we're invalidating a particular codeinstance, only invalidate + // this backedge it actually has an edge for our codeinstance. + jl_svec_t *edges = jl_atomic_load_relaxed(&replaced->edges); + for (size_t j = 0; j < jl_svec_len(edges); ++j) { + jl_value_t *edge = jl_svecref(edges, j); + if (edge == (jl_value_t*)replaced_mi || edge == (jl_value_t*)replaced_ci) + goto found; + } + // Keep this entry in the backedge list, but compact it + ins = set_next_edge(backedges, ins, invokesig, replaced); + continue; + found:; + } invalidate_code_instance(replaced, max_world, depth); } + if (replaced_ci && ins != 0) { + jl_array_del_end(backedges, l - ins); + // If we're only invalidating one ci, we don't know which ci any particular + // backedge was for, so we can't delete them. Put them back. + replaced_mi->backedges = backedges; + jl_gc_wb(replaced_mi, backedges); + } JL_GC_POP(); } } @@ -1894,7 +1919,7 @@ static void _invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_ static void invalidate_backedges(jl_method_instance_t *replaced_mi, size_t max_world, const char *why) { JL_LOCK(&replaced_mi->def.method->writelock); - _invalidate_backedges(replaced_mi, max_world, 1); + _invalidate_backedges(replaced_mi, NULL, max_world, 1); JL_UNLOCK(&replaced_mi->def.method->writelock); if (why && _jl_debug_method_invalidation) { jl_array_ptr_1d_push(_jl_debug_method_invalidation, (jl_value_t*)replaced_mi); @@ -1928,8 +1953,8 @@ JL_DLLEXPORT void jl_method_instance_add_backedge(jl_method_instance_t *callee, size_t i = 0, l = jl_array_nrows(callee->backedges); for (i = 0; i < l; i++) { // optimized version of while (i < l) i = get_next_edge(callee->backedges, i, &invokeTypes, &mi); - jl_value_t *mi = jl_array_ptr_ref(callee->backedges, i); - if (mi != (jl_value_t*)caller) + jl_value_t *ciedge = jl_array_ptr_ref(callee->backedges, i); + if (ciedge != (jl_value_t*)caller) continue; jl_value_t *invokeTypes = i > 0 ? jl_array_ptr_ref(callee->backedges, i - 1) : NULL; if (invokeTypes && jl_is_method_instance(invokeTypes)) @@ -2372,7 +2397,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) continue; loctag = jl_atomic_load_relaxed(&m->specializations); // use loctag for a gcroot _Atomic(jl_method_instance_t*) *data; - size_t i, l; + size_t l; if (jl_is_svec(loctag)) { data = (_Atomic(jl_method_instance_t*)*)jl_svec_data(loctag); l = jl_svec_len(loctag); @@ -2382,7 +2407,7 @@ void jl_method_table_activate(jl_methtable_t *mt, jl_typemap_entry_t *newentry) l = 1; } enum morespec_options ambig = morespec_unknown; - for (i = 0; i < l; i++) { + for (size_t i = 0; i < l; i++) { jl_method_instance_t *mi = jl_atomic_load_relaxed(&data[i]); if ((jl_value_t*)mi == jl_nothing) continue; diff --git a/src/jltypes.c b/src/jltypes.c index 2af9340d5bdef..e6d48b6f489b2 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3275,7 +3275,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_svec(5, jl_any_type/*jl_globalref_type*/, jl_any_type, jl_binding_partition_type, jl_any_type, jl_uint8_type), jl_emptysvec, 0, 1, 0); - const static uint32_t binding_atomicfields[] = { 0x0005 }; // Set fields 2, 3 as atomic + const static uint32_t binding_atomicfields[] = { 0x0016 }; // Set fields 2, 3, 5 as atomic jl_binding_type->name->atomicfields = binding_atomicfields; const static uint32_t binding_constfields[] = { 0x0001 }; // Set fields 1 as constant jl_binding_type->name->constfields = binding_constfields; @@ -3539,7 +3539,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_method_type = jl_new_datatype(jl_symbol("Method"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(31, + jl_perm_symsvec(32, "name", "module", "file", @@ -3568,10 +3568,11 @@ void jl_init_types(void) JL_GC_DISABLED "isva", "is_for_opaque_closure", "nospecializeinfer", + "did_scan_source", "constprop", "max_varargs", "purity"), - jl_svec(31, + jl_svec(32, jl_symbol_type, jl_module_type, jl_symbol_type, @@ -3602,6 +3603,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_bool_type, jl_uint8_type, jl_uint8_type, + jl_uint8_type, jl_uint16_type), jl_emptysvec, 0, 1, 10); diff --git a/src/julia.h b/src/julia.h index 4d311d84366d9..0dd9ba7027e98 100644 --- a/src/julia.h +++ b/src/julia.h @@ -374,6 +374,10 @@ typedef struct _jl_method_t { uint8_t isva; uint8_t is_for_opaque_closure; uint8_t nospecializeinfer; + // bit flags, 0x01 = scanned + // 0x02 = added to module scanned list (either from scanning or inference edge) + _Atomic(uint8_t) did_scan_source; + // uint8 settings uint8_t constprop; // 0x00 = use heuristic; 0x01 = aggressive; 0x02 = none uint8_t max_varargs; // 0xFF = use heuristic; otherwise, max # of args to expand @@ -750,7 +754,10 @@ enum jl_binding_flags { BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION = 0x1, BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION = 0x2, // `export` is tracked in partitions, but sets this as well - BINDING_FLAG_PUBLICP = 0x4 + BINDING_FLAG_PUBLICP = 0x4, + // Set if any methods defined in this module implicitly reference + // this binding. If not, invalidation is optimized. + BINDING_FLAG_ANY_IMPLICIT_EDGES = 0x8 }; typedef struct _jl_binding_t { @@ -776,6 +783,7 @@ typedef struct _jl_module_t { jl_sym_t *file; int32_t line; jl_value_t *usings_backedges; + jl_value_t *scanned_methods; // hidden fields: arraylist_t usings; /* arraylist of struct jl_module_using */ // modules with all bindings potentially imported jl_uuid_t build_id; @@ -2067,6 +2075,7 @@ JL_DLLEXPORT int jl_get_module_infer(jl_module_t *m); JL_DLLEXPORT void jl_set_module_max_methods(jl_module_t *self, int value); JL_DLLEXPORT int jl_get_module_max_methods(jl_module_t *m); JL_DLLEXPORT jl_value_t *jl_get_module_usings_backedges(jl_module_t *m); +JL_DLLEXPORT jl_value_t *jl_get_module_scanned_methods(jl_module_t *m); JL_DLLEXPORT jl_value_t *jl_get_module_binding_or_nothing(jl_module_t *m, jl_sym_t *s); // get binding for reading diff --git a/src/julia_internal.h b/src/julia_internal.h index ff11926fe42c9..a5b7be7bba0b6 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -722,7 +722,7 @@ jl_code_info_t *jl_new_code_info_from_ir(jl_expr_t *ast); JL_DLLEXPORT jl_code_info_t *jl_new_code_info_uninit(void); JL_DLLEXPORT void jl_resolve_definition_effects_in_ir(jl_array_t *stmts, jl_module_t *m, jl_svec_t *sparam_vals, jl_value_t *binding_edge, int binding_effects); -JL_DLLEXPORT void jl_maybe_add_binding_backedge(jl_globalref_t *gr, jl_module_t *defining_module, jl_value_t *edge); +JL_DLLEXPORT int jl_maybe_add_binding_backedge(jl_binding_t *b, jl_value_t *edge, jl_method_t *in_method); JL_DLLEXPORT void jl_add_binding_backedge(jl_binding_t *b, jl_value_t *edge); int get_next_edge(jl_array_t *list, int i, jl_value_t** invokesig, jl_code_instance_t **caller) JL_NOTSAFEPOINT; @@ -878,6 +878,7 @@ STATIC_INLINE size_t module_usings_max(jl_module_t *m) JL_NOTSAFEPOINT { } JL_DLLEXPORT jl_sym_t *jl_module_name(jl_module_t *m) JL_NOTSAFEPOINT; +void jl_add_scanned_method(jl_module_t *m, jl_method_t *meth); jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e); jl_value_t *jl_interpret_opaque_closure(jl_opaque_closure_t *clos, jl_value_t **args, size_t nargs); jl_value_t *jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t *src); diff --git a/src/method.c b/src/method.c index ffd33645310f3..11609f9d3b61d 100644 --- a/src/method.c +++ b/src/method.c @@ -39,6 +39,45 @@ static void check_c_types(const char *where, jl_value_t *rt, jl_value_t *at) } } +void jl_add_scanned_method(jl_module_t *m, jl_method_t *meth) +{ + JL_LOCK(&m->lock); + if (m->scanned_methods == jl_nothing) { + m->scanned_methods = (jl_value_t*)jl_alloc_vec_any(0); + jl_gc_wb(m, m->scanned_methods); + } + jl_array_ptr_1d_push((jl_array_t*)m->scanned_methods, (jl_value_t*)meth); + JL_UNLOCK(&m->lock); +} + +JL_DLLEXPORT void jl_scan_method_source_now(jl_method_t *m, jl_value_t *src) +{ + if (!jl_atomic_fetch_or(&m->did_scan_source, 1)) { + jl_code_info_t *code = NULL; + JL_GC_PUSH1(&code); + if (!jl_is_code_info(src)) + code = jl_uncompress_ir(m, NULL, src); + else + code = (jl_code_info_t*)src; + jl_array_t *stmts = code->code; + size_t i, l = jl_array_nrows(stmts); + int any_implicit = 0; + for (i = 0; i < l; i++) { + jl_value_t *stmt = jl_array_ptr_ref(stmts, i); + if (jl_is_globalref(stmt)) { + jl_globalref_t *gr = (jl_globalref_t*)stmt; + jl_binding_t *b = gr->binding; + if (!b) + b = jl_get_module_binding(gr->mod, gr->name, 1); + any_implicit |= jl_maybe_add_binding_backedge(b, (jl_value_t*)m, m); + } + } + if (any_implicit && !(jl_atomic_fetch_or(&m->did_scan_source, 0x2) & 0x2)) + jl_add_scanned_method(m->module, m); + JL_GC_POP(); + } +} + // Resolve references to non-locally-defined variables to become references to global // variables in `module` (unless the rvalue is one of the type parameters in `sparam_vals`). static jl_value_t *resolve_definition_effects(jl_value_t *expr, jl_module_t *module, jl_svec_t *sparam_vals, jl_value_t *binding_edge, @@ -47,10 +86,7 @@ static jl_value_t *resolve_definition_effects(jl_value_t *expr, jl_module_t *mod if (jl_is_symbol(expr)) { jl_error("Found raw symbol in code returned from lowering. Expected all symbols to have been resolved to GlobalRef or slots."); } - if (jl_is_globalref(expr)) { - jl_maybe_add_binding_backedge((jl_globalref_t*)expr, module, binding_edge); - return expr; - } + if (!jl_is_expr(expr)) { return expr; } @@ -973,6 +1009,7 @@ JL_DLLEXPORT jl_method_t *jl_new_method_uninit(jl_module_t *module) jl_atomic_store_relaxed(&m->deleted_world, 1); m->is_for_opaque_closure = 0; m->nospecializeinfer = 0; + jl_atomic_store_relaxed(&m->did_scan_source, 0); m->constprop = 0; m->purity.bits = 0; m->max_varargs = UINT8_MAX; diff --git a/src/module.c b/src/module.c index 8f5af51da7668..572f5fd7e8142 100644 --- a/src/module.c +++ b/src/module.c @@ -106,6 +106,9 @@ void jl_check_new_binding_implicit( if (tempbmax_world < max_world) max_world = tempbmax_world; + // N.B.: Which aspects of the partition are considered here needs to + // be kept in sync with `export_affecting_partition_flags` in the + // invalidation code. if ((tempbpart->kind & PARTITION_FLAG_EXPORTED) == 0) continue; @@ -165,6 +168,29 @@ void jl_check_new_binding_implicit( return; } +JL_DLLEXPORT jl_binding_partition_t *jl_maybe_reresolve_implicit(jl_binding_t *b, size_t new_max_world) +{ + jl_binding_partition_t *new_bpart = new_binding_partition(); + jl_binding_partition_t *bpart = jl_atomic_load_acquire(&b->partitions); + assert(bpart); + while (1) { + jl_atomic_store_relaxed(&new_bpart->next, bpart); + jl_gc_wb(new_bpart, bpart); + jl_check_new_binding_implicit(new_bpart, b, NULL, new_max_world+1); + JL_GC_PROMISE_ROOTED(new_bpart); // TODO: Analyzer doesn't understand MAYBE_UNROOTED properly + if (bpart->kind & PARTITION_FLAG_EXPORTED) + new_bpart->kind |= PARTITION_FLAG_EXPORTED; + if (new_bpart->kind == bpart->kind && new_bpart->restriction == bpart->restriction) + return bpart; + // Resolution changed, insert the new partition + size_t expected_max_world = ~(size_t)0; + if (jl_atomic_cmpswap(&bpart->max_world, &expected_max_world, new_max_world) && + jl_atomic_cmpswap(&b->partitions, &bpart, new_bpart)) + break; + } + return new_bpart; +} + STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, jl_value_t *parent, _Atomic(jl_binding_partition_t *)*insert, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED { assert(jl_is_binding(b)); @@ -183,7 +209,8 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b if (!new_bpart) new_bpart = new_binding_partition(); jl_atomic_store_relaxed(&new_bpart->next, bpart); - jl_gc_wb_fresh(new_bpart, bpart); + if (bpart) + jl_gc_wb(new_bpart, bpart); // Not fresh the second time around the loop new_bpart->min_world = bpart ? jl_atomic_load_relaxed(&bpart->max_world) + 1 : 0; jl_atomic_store_relaxed(&new_bpart->max_world, max_world); JL_GC_PROMISE_ROOTED(new_bpart); // TODO: Analyzer doesn't understand MAYBE_UNROOTED properly @@ -293,6 +320,7 @@ JL_DLLEXPORT jl_module_t *jl_new_module__(jl_sym_t *name, jl_module_t *parent) m->build_id.hi = ~(uint64_t)0; jl_atomic_store_relaxed(&m->counter, 1); m->usings_backedges = jl_nothing; + m->scanned_methods = jl_nothing; m->nospecialize = 0; m->optlevel = -1; m->compile = -1; @@ -1112,10 +1140,12 @@ JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from) jl_sym_t *var = b->globalref->name; jl_binding_t *tob = jl_get_module_binding(to, var, 0); if (tob) { - jl_binding_partition_t *tobpart = jl_get_binding_partition(tob, new_world); - enum jl_partition_kind kind = jl_binding_kind(tobpart); - if (jl_bkind_is_some_implicit(kind)) { - jl_replace_binding_locked(tob, tobpart, NULL, PARTITION_KIND_IMPLICIT_RECOMPUTE, new_world); + jl_binding_partition_t *tobpart = jl_atomic_load_relaxed(&tob->partitions); + if (tobpart) { + enum jl_partition_kind kind = jl_binding_kind(tobpart); + if (jl_bkind_is_some_implicit(kind)) { + jl_replace_binding_locked(tob, tobpart, NULL, PARTITION_KIND_IMPLICIT_RECOMPUTE, new_world); + } } } } @@ -1135,6 +1165,25 @@ JL_DLLEXPORT jl_value_t *jl_get_module_usings_backedges(jl_module_t *m) return m->usings_backedges; } +JL_DLLEXPORT size_t jl_module_scanned_methods_length(jl_module_t *m) +{ + JL_LOCK(&m->lock); + size_t len = 0; + if (m->scanned_methods != jl_nothing) + len = jl_array_len(m->scanned_methods); + JL_UNLOCK(&m->lock); + return len; +} + +JL_DLLEXPORT jl_value_t *jl_module_scanned_methods_getindex(jl_module_t *m, size_t i) +{ + JL_LOCK(&m->lock); + assert(m->scanned_methods != jl_nothing); + jl_value_t *ret = jl_array_ptr_ref(m->scanned_methods, i-1); + JL_UNLOCK(&m->lock); + return ret; +} + JL_DLLEXPORT jl_value_t *jl_get_module_binding_or_nothing(jl_module_t *m, jl_sym_t *s) { jl_binding_t *b = jl_get_module_binding(m, s, 0); @@ -1341,20 +1390,22 @@ JL_DLLEXPORT void jl_add_binding_backedge(jl_binding_t *b, jl_value_t *edge) // Called for all GlobalRefs found in lowered code. Adds backedges for cross-module // GlobalRefs. -JL_DLLEXPORT void jl_maybe_add_binding_backedge(jl_globalref_t *gr, jl_module_t *defining_module, jl_value_t *edge) +JL_DLLEXPORT int jl_maybe_add_binding_backedge(jl_binding_t *b, jl_value_t *edge, jl_method_t *for_method) { if (!edge) - return; + return 0; + jl_module_t *defining_module = for_method->module; // N.B.: The logic for evaluating whether a backedge is required must // match the invalidation logic. - if (gr->mod == defining_module) { + if (b->globalref->mod == defining_module) { // No backedge required - invalidation will forward scan - return; + jl_atomic_fetch_or(&b->flags, BINDING_FLAG_ANY_IMPLICIT_EDGES); + if (!(jl_atomic_fetch_or(&for_method->did_scan_source, 0x2) & 0x2)) + jl_add_scanned_method(for_method->module, for_method); + return 1; } - jl_binding_t *b = gr->binding; - if (!b) - b = jl_get_module_binding(gr->mod, gr->name, 1); - jl_add_binding_backedge(b, edge); + jl_add_binding_backedge(b, (jl_value_t*)edge); + return 0; } JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked(jl_binding_t *b, @@ -1379,7 +1430,6 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, jl_atomic_store_relaxed(&jl_first_image_replacement_world, new_world); assert(jl_atomic_load_relaxed(&b->partitions) == old_bpart); - jl_atomic_store_release(&old_bpart->max_world, new_world-1); jl_binding_partition_t *new_bpart = new_binding_partition(); JL_GC_PUSH1(&new_bpart); new_bpart->min_world = new_world; @@ -1387,12 +1437,17 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, assert(!restriction_val); jl_check_new_binding_implicit(new_bpart /* callee rooted */, b, NULL, new_world); new_bpart->kind |= kind & PARTITION_MASK_FLAG; + if (new_bpart->kind == old_bpart->kind && new_bpart->restriction == old_bpart->restriction) { + JL_GC_POP(); + return old_bpart; + } } else { new_bpart->kind = kind; new_bpart->restriction = restriction_val; jl_gc_wb_fresh(new_bpart, restriction_val); } + jl_atomic_store_release(&old_bpart->max_world, new_world-1); jl_atomic_store_relaxed(&new_bpart->next, old_bpart); jl_gc_wb_fresh(new_bpart, old_bpart); diff --git a/src/staticdata.c b/src/staticdata.c index d9c2912673708..724019fbe1559 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -836,6 +836,7 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ } jl_queue_for_serialization(s, m->usings_backedges); + jl_queue_for_serialization(s, m->scanned_methods); } // Anything that requires uniquing or fixing during deserialization needs to be "toplevel" @@ -1348,6 +1349,9 @@ static void jl_write_module(jl_serializer_state *s, uintptr_t item, jl_module_t newm->usings_backedges = NULL; arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, usings_backedges))); arraylist_push(&s->relocs_list, (void*)backref_id(s, m->usings_backedges, s->link_ids_relocs)); + newm->scanned_methods = NULL; + arraylist_push(&s->relocs_list, (void*)(reloc_offset + offsetof(jl_module_t, scanned_methods))); + arraylist_push(&s->relocs_list, (void*)backref_id(s, m->scanned_methods, s->link_ids_relocs)); // After reload, everything that has happened in this process happened semantically at // (for .incremental) or before jl_require_world, so reset this flag. diff --git a/test/rebinding.jl b/test/rebinding.jl index 8f54aa99b15bd..ab9696c7f0222 100644 --- a/test/rebinding.jl +++ b/test/rebinding.jl @@ -298,3 +298,23 @@ module RangeMerge @test !contains(get_llvm(f, Tuple{}), "jl_get_binding_value") end + +# Test that we invalidate for undefined -> defined transitions (#54733) +module UndefinedTransitions + using Test + function foo54733() + for i = 1:1_000_000_000 + bar54733(i) + end + return 1 + end + @test_throws UndefVarError foo54733() + let ci = first(methods(foo54733)).specializations.cache + @test !Base.Compiler.is_nothrow(Base.Compiler.decode_effects(ci.ipo_purity_bits)) + end + bar54733(x) = 3x + @test foo54733() === 1 + let ci = first(methods(foo54733)).specializations.cache + @test Base.Compiler.is_nothrow(Base.Compiler.decode_effects(ci.ipo_purity_bits)) + end +end From 10f5d446b502c5072b80d2ce854ab1f6419b7bc8 Mon Sep 17 00:00:00 2001 From: Christian Guinard <28689358+christiangnrd@users.noreply.github.com> Date: Sat, 8 Mar 2025 12:13:14 -0400 Subject: [PATCH 28/74] [1.12] Add PR number to `gc_safe` NEWS entry (#57682) 1.12 branch version of #57679 --- NEWS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEWS.md b/NEWS.md index 30782942fd612..615b495018c18 100644 --- a/NEWS.md +++ b/NEWS.md @@ -25,7 +25,7 @@ New language features * Support for Unicode 16 ([#56925]). * `Threads.@spawn` now takes a `:samepool` argument to specify the same threadpool as the caller. `Threads.@spawn :samepool foo()` which is shorthand for `Threads.@spawn Threads.threadpool() foo()` ([#57109]). -* The `@ccall` macro can now take a `gc_safe` argument, that if set to true allows the runtime to run garbage collection concurrently to the `ccall` +* The `@ccall` macro can now take a `gc_safe` argument, that if set to true allows the runtime to run garbage collection concurrently to the `ccall` ([#49933]). Language changes ---------------- From 8d76a6736882ddc72fbe9c61b9f8de8b1abcfce5 Mon Sep 17 00:00:00 2001 From: Junfeng Qiao Date: Wed, 5 Mar 2025 11:42:42 +0100 Subject: [PATCH 29/74] Add missing arg in TOML `printvalue` (#57584) Otherwise error when calling `print_inline_table` (cherry picked from commit f20722476de363e55c8d3a7775ef6fbdcae5d5ef) --- stdlib/TOML/src/print.jl | 4 ++-- stdlib/TOML/test/print.jl | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/stdlib/TOML/src/print.jl b/stdlib/TOML/src/print.jl index 63f65b017d393..c6c046b9b40c6 100644 --- a/stdlib/TOML/src/print.jl +++ b/stdlib/TOML/src/print.jl @@ -77,7 +77,7 @@ end # Fallback function printvalue(f::MbyFunc, io::IO, value, sorted::Bool) toml_value = to_toml_value(f, value) - @invokelatest printvalue(f, io, toml_value) + @invokelatest printvalue(f, io, toml_value, sorted) end function printvalue(f::MbyFunc, io::IO, value::AbstractVector, sorted::Bool) @@ -156,7 +156,7 @@ function print_table(f::MbyFunc, io::IO, a::AbstractDict, ) if a in inline_tables - @invokelatest print_inline_table(f, io, a) + @invokelatest print_inline_table(f, io, a, sorted) return end diff --git a/stdlib/TOML/test/print.jl b/stdlib/TOML/test/print.jl index 8fba1b1c1df10..e8a6431cb34a7 100644 --- a/stdlib/TOML/test/print.jl +++ b/stdlib/TOML/test/print.jl @@ -94,6 +94,14 @@ loaders = ["gzip", { driver = "csv", args = {delim = "\t"}}] a = 222 d = 333 """ + + # https://github.com/JuliaLang/julia/pull/57584 + d = Dict("b" => [MyStruct(1), MyStruct(2)]) + @test toml_str(d) do x + x isa MyStruct && return Dict("a" => x.a) + end == """ + b = [{a = 1}, {a = 2}] + """ end @testset "unsigned integers" for (x, s) in [ @@ -196,6 +204,14 @@ LocalPkg = {path = "LocalPkg"} @test toml_str(d; sorted=true, inline_tables) == s @test roundtrip(s) + +# https://github.com/JuliaLang/julia/pull/57584 +d = Dict("a" => 1, "b" => 2) +inline_tables = IdSet{Dict}([d]) +s = "{a = 1, b = 2}" +@test toml_str(d; sorted=true, inline_tables) == s + + # multiline strings (#55083) s = """ a = \"\"\"lorem ipsum From cf6971752a418157af9c25b02feeb800f2dfff63 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Wed, 5 Mar 2025 20:58:10 -0500 Subject: [PATCH 30/74] Compiler: Fix pre-compilation as separate package (#57650) As spotted by @keno in https://github.com/JuliaLang/julia/pull/57530#issuecomment-2689772500 (cherry picked from commit a001847c3eba84e8e5ef93dc7a52e90f320cb1dd) --- Compiler/src/Compiler.jl | 27 +++++++++++++-------------- Compiler/src/typeinfer.jl | 1 - Compiler/src/verifytrim.jl | 1 - 3 files changed, 13 insertions(+), 16 deletions(-) diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index c75a524ca40ec..f28bc02301dfd 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -192,23 +192,22 @@ end module IRShow end # relies on string and IO operations defined in Base baremodule TrimVerifier end # relies on IRShow, so define this afterwards -function load_irshow!() - if isdefined(Base, :end_base_include) - # This code path is exclusively for Revise, which may want to re-run this - # after bootstrap. - Compilerdir = Base.dirname(Base.String(@__SOURCE_FILE__)) - include(IRShow, Base.joinpath(Compilerdir, "ssair/show.jl")) - include(TrimVerifier, Base.joinpath(Compilerdir, "verifytrim.jl")) - else +if isdefined(Base, :end_base_include) + # When this module is loaded as the standard library, include these files as usual + include(IRShow, "ssair/show.jl") + include(TrimVerifier, "verifytrim.jl") +else + function load_irshow!() + Base.delete_method(Base.which(verify_typeinf_trim, (IO, Vector{Any}, Bool)),) include(IRShow, "ssair/show.jl") include(TrimVerifier, "verifytrim.jl") end -end -if !isdefined(Base, :end_base_include) - # During bootstrap, skip including this file and defer it to base/show.jl to include later -else - # When this module is loaded as the standard library, include this file as usual - load_irshow!() + function verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) + # stub implementation + msg = "--trim verifier not defined" + onlywarn ? println(io, msg) : error(msg) + end + # During bootstrap, skip including these files and defer to base/show.jl to include it later end end # baremodule Compiler diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 7b0cf09601c4a..fb44bc7cc8989 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -1350,7 +1350,6 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m return codeinfos end -verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) = (msg = "--trim verifier not defined"; onlywarn ? println(io, msg) : error(msg)) verify_typeinf_trim(codeinfos::Vector{Any}, onlywarn::Bool) = invokelatest(verify_typeinf_trim, stdout, codeinfos, onlywarn) function return_type(@nospecialize(f), t::DataType) # this method has a special tfunc diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index 67728a15177b4..36f9413a8c1a7 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -310,7 +310,6 @@ end # driver / verifier implemented by juliac-buildscript.jl for the purpose of extensibility. # For now, it is part of Base.Compiler, but executed with invokelatest so that packages # could provide hooks to change, customize, or tweak its behavior and heuristics. -Base.delete_method(Base.which(verify_typeinf_trim, (IO, Vector{Any}, Bool)),) function verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) errors, parents = get_verify_typeinf_trim(codeinfos) From 1679d93fcaea70c7eb7d73af2af40f47ce988e5e Mon Sep 17 00:00:00 2001 From: N5N3 <2642243996@qq.com> Date: Thu, 6 Mar 2025 21:25:41 +0800 Subject: [PATCH 31/74] Intersect: try normal+reverse+existential subtyping during intersection (#57476) Typevars are all existential (in the sense of variable lb/ub) during intersection. This fix is somehow costly as we have to perform 3 times check to prove a false result. But a single existential <: seems too dangerous as it cause many circular env in the past. fix #57429 fix #41561 (cherry picked from commit beb928bfec532c8f038e113df2960c7b9609edb6) --- src/subtype.c | 127 ++++++++++++++++++++++++++++++++++++------------ test/subtype.jl | 33 ++++++++----- 2 files changed, 118 insertions(+), 42 deletions(-) diff --git a/src/subtype.c b/src/subtype.c index a0b7bff4006ce..1086d11d2fe8c 100644 --- a/src/subtype.c +++ b/src/subtype.c @@ -2678,31 +2678,22 @@ static void set_bound(jl_value_t **bound, jl_value_t *val, jl_tvar_t *v, jl_sten // subtype, treating all vars as existential static int subtype_in_env_existential(jl_value_t *x, jl_value_t *y, jl_stenv_t *e) { - jl_varbinding_t *v = e->vars; - int len = 0; if (x == jl_bottom_type || y == (jl_value_t*)jl_any_type) return 1; - while (v != NULL) { - len++; - v = v->prev; - } - int8_t *rs = (int8_t*)malloc_s(len); + int8_t *rs = (int8_t*)alloca(current_env_length(e)); + jl_varbinding_t *v = e->vars; int n = 0; - v = e->vars; - while (n < len) { - assert(v != NULL); + while (v != NULL) { rs[n++] = v->right; v->right = 1; v = v->prev; } int issub = subtype_in_env(x, y, e); n = 0; v = e->vars; - while (n < len) { - assert(v != NULL); + while (v != NULL) { v->right = rs[n++]; v = v->prev; } - free(rs); return issub; } @@ -2750,6 +2741,8 @@ static int check_unsat_bound(jl_value_t *t, jl_tvar_t *v, jl_stenv_t *e) JL_NOTS } +static int intersect_var_ccheck_in_env(jl_value_t *xlb, jl_value_t *xub, jl_value_t *ylb, jl_value_t *yub, jl_stenv_t *e, int flip); + static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int8_t R, int param) { jl_varbinding_t *bb = lookup(e, b); @@ -2761,20 +2754,14 @@ static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int return R ? intersect(a, bb->lb, e, param) : intersect(bb->lb, a, e, param); if (!jl_is_type(a) && !jl_is_typevar(a)) return set_var_to_const(bb, a, e, R); - jl_savedenv_t se; if (param == 2) { jl_value_t *ub = NULL; JL_GC_PUSH1(&ub); if (!jl_has_free_typevars(a)) { - save_env(e, &se, 1); - int issub = subtype_in_env_existential(bb->lb, a, e); - restore_env(e, &se, 1); - if (issub) { - issub = subtype_in_env_existential(a, bb->ub, e); - restore_env(e, &se, 1); - } - free_env(&se); - if (!issub) { + if (R) flip_offset(e); + int ccheck = intersect_var_ccheck_in_env(bb->lb, bb->ub, a, a, e, !R); + if (R) flip_offset(e); + if (!ccheck) { JL_GC_POP(); return jl_bottom_type; } @@ -2784,6 +2771,7 @@ static jl_value_t *intersect_var(jl_tvar_t *b, jl_value_t *a, jl_stenv_t *e, int e->triangular++; ub = R ? intersect_aside(a, bb->ub, e, bb->depth0) : intersect_aside(bb->ub, a, e, bb->depth0); e->triangular--; + jl_savedenv_t se; save_env(e, &se, 1); int issub = subtype_in_env_existential(bb->lb, ub, e); restore_env(e, &se, 1); @@ -3856,6 +3844,89 @@ static int subtype_by_bounds(jl_value_t *x, jl_value_t *y, jl_stenv_t *e) JL_NOT return compareto_var(x, (jl_tvar_t*)y, e, -1) || compareto_var(y, (jl_tvar_t*)x, e, 1); } +static int intersect_var_ccheck_in_env(jl_value_t *xlb, jl_value_t *xub, jl_value_t *ylb, jl_value_t *yub, jl_stenv_t *e, int flip) +{ + int easy_check1 = xlb == jl_bottom_type || + yub == (jl_value_t *)jl_any_type || + (e->Loffset == 0 && obviously_in_union(yub, xlb)); + int easy_check2 = ylb == jl_bottom_type || + xub == (jl_value_t *)jl_any_type || + (e->Loffset == 0 && obviously_in_union(xub, ylb)); + int nofree1 = 0, nofree2 = 0; + if (!easy_check1) { + nofree1 = !jl_has_free_typevars(xlb) && !jl_has_free_typevars(yub); + if (nofree1 && e->Loffset == 0) { + easy_check1 = jl_subtype(xlb, yub); + if (!easy_check1) + return 0; + } + } + if (!easy_check2) { + nofree2 = !jl_has_free_typevars(ylb) && !jl_has_free_typevars(xub); + if (nofree2 && e->Loffset == 0) { + easy_check2 = jl_subtype(ylb, xub); + if (!easy_check2) + return 0; + } + } + if (easy_check1 && easy_check2) + return 1; + int ccheck = 0; + if ((easy_check1 || nofree1) && (easy_check2 || nofree2)) { + jl_varbinding_t *vars = e->vars; + e->vars = NULL; + ccheck = easy_check1 || subtype_in_env(xlb, yub, e); + if (ccheck && !easy_check2) { + flip_offset(e); + ccheck = subtype_in_env(ylb, xub, e); + flip_offset(e); + } + e->vars = vars; + return ccheck; + } + jl_savedenv_t se; + save_env(e, &se, 1); + // first try normal flip. + if (flip) flip_vars(e); + ccheck = easy_check1 || subtype_in_env(xlb, yub, e); + if (ccheck && !easy_check2) { + flip_offset(e); + ccheck = subtype_in_env(ylb, xub, e); + flip_offset(e); + } + if (flip) flip_vars(e); + if (!ccheck) { + // then try reverse flip. + restore_env(e, &se, 1); + if (!flip) flip_vars(e); + ccheck = easy_check1 || subtype_in_env(xlb, yub, e); + if (ccheck && !easy_check2) { + flip_offset(e); + ccheck = subtype_in_env(ylb, xub, e); + flip_offset(e); + } + if (!flip) flip_vars(e); + } + if (!ccheck) { + // then try existential. + restore_env(e, &se, 1); + if (easy_check1) + ccheck = 1; + else { + ccheck = subtype_in_env_existential(xlb, yub, e); + restore_env(e, &se, 1); + } + if (ccheck && !easy_check2) { + flip_offset(e); + ccheck = subtype_in_env_existential(ylb, xub, e); + flip_offset(e); + restore_env(e, &se, 1); + } + } + free_env(&se); + return ccheck; +} + static int has_typevar_via_env(jl_value_t *x, jl_tvar_t *t, jl_stenv_t *e) { if (e->Loffset == 0) { @@ -3988,14 +4059,8 @@ static jl_value_t *intersect(jl_value_t *x, jl_value_t *y, jl_stenv_t *e, int pa ccheck = 1; } else { - if (R) flip_vars(e); - ccheck = subtype_in_env(xlb, yub, e); - if (ccheck) { - flip_offset(e); - ccheck = subtype_in_env(ylb, xub, e); - flip_offset(e); - } - if (R) flip_vars(e); + // try many subtype check to avoid false `Union{}` + ccheck = intersect_var_ccheck_in_env(xlb, xub, ylb, yub, e, R); } if (R) flip_offset(e); if (!ccheck) diff --git a/test/subtype.jl b/test/subtype.jl index ba7f86bb86a14..979746bd626dc 100644 --- a/test/subtype.jl +++ b/test/subtype.jl @@ -1691,9 +1691,7 @@ CovType{T} = Union{AbstractArray{T,2}, # issue #31703 @testintersect(Pair{<:Any, Ref{Tuple{Ref{Ref{Tuple{Int}}},Ref{Float64}}}}, Pair{T, S} where S<:(Ref{A} where A<:(Tuple{C,Ref{T}} where C<:(Ref{D} where D<:(Ref{E} where E<:Tuple{FF}) where FF<:B)) where B) where T, - Pair{T, Ref{Tuple{Ref{Ref{Tuple{Int}}},Ref{Float64}}}} where T) -# TODO: should be able to get this result -# Pair{Float64, Ref{Tuple{Ref{Ref{Tuple{Int}}},Ref{Float64}}}} + Pair{Float64, Ref{Tuple{Ref{Ref{Tuple{Int}}},Ref{Float64}}}}) module I31703 using Test, LinearAlgebra @@ -1745,8 +1743,7 @@ end Tuple{Type{SA{2, L}}, Type{SA{2, L}}} where L) @testintersect(Tuple{Type{SA{2, L}}, Type{SA{2, 16}}} where L, Tuple{Type{<:SA{N, L}}, Type{<:SA{N, L}}} where {N,L}, - # TODO: this could be narrower - Tuple{Type{SA{2, L}}, Type{SA{2, 16}}} where L) + Tuple{Type{SA{2, 16}}, Type{SA{2, 16}}}) # issue #31993 @testintersect(Tuple{Type{<:AbstractVector{T}}, Int} where T, @@ -1851,9 +1848,9 @@ c32703(::Type{<:Str{C}}, str::Str{C}) where {C<:CSE} = str Tuple{Type{<:Str{C}}, Str{C}} where {C<:CSE}, Union{}) @test c32703(UTF16Str, ASCIIStr()) == 42 -@test_broken typeintersect(Tuple{Vector{Vector{Float32}},Matrix,Matrix}, - Tuple{Vector{V},Matrix{Int},Matrix{S}} where {S, V<:AbstractVector{S}}) == - Tuple{Array{Array{Float32,1},1},Array{Int,2},Array{Float32,2}} +@testintersect(Tuple{Vector{Vector{Float32}},Matrix,Matrix}, + Tuple{Vector{V},Matrix{Int},Matrix{S}} where {S, V<:AbstractVector{S}}, + Tuple{Array{Array{Float32,1},1},Array{Int,2},Array{Float32,2}}) @testintersect(Tuple{Pair{Int, DataType}, Any}, Tuple{Pair{A, B} where B<:Type, Int} where A, @@ -2469,6 +2466,11 @@ end abstract type P47654{A} end @test Wrapper47654{P47654, Vector{Union{P47654,Nothing}}} <: Wrapper47654 +#issue 41561 +@testintersect(Tuple{Vector{VT}, Vector{VT}} where {N1, VT<:AbstractVector{N1}}, + Tuple{Vector{VN} where {N, VN<:AbstractVector{N}}, Vector{Vector{Float64}}}, + Tuple{Vector{Vector{Float64}}, Vector{Vector{Float64}}}) + @testset "known subtype/intersect issue" begin #issue 45874 let S = Pair{Val{P}, AbstractVector{<:Union{P,<:AbstractMatrix{P}}}} where P, @@ -2476,9 +2478,6 @@ abstract type P47654{A} end @test S <: T end - #issue 41561 - @test_broken typeintersect(Tuple{Vector{VT}, Vector{VT}} where {N1, VT<:AbstractVector{N1}}, - Tuple{Vector{VN} where {N, VN<:AbstractVector{N}}, Vector{Vector{Float64}}}) !== Union{} #issue 40865 @test Tuple{Set{Ref{Int}}, Set{Ref{Int}}} <: Tuple{Set{KV}, Set{K}} where {K,KV<:Union{K,Ref{K}}} @test Tuple{Set{Val{Int}}, Set{Val{Int}}} <: Tuple{Set{KV}, Set{K}} where {K,KV<:Union{K,Val{K}}} @@ -2746,3 +2745,15 @@ end Val{Tuple{T,R,S}} where {T,R<:Vector{T},S<:Vector{R}}, Val{Tuple{Int, Vector{Int}, T}} where T<:Vector{Vector{Int}}, ) + +#issue 57429 +@testintersect( + Pair{<:Any, <:Tuple{Int}}, + Pair{N, S} where {N, NTuple{N,Int}<:S<:NTuple{M,Int} where {M}}, + !Union{} +) +@testintersect( + Pair{N, T} where {N,NTuple{N,Int}<:T<:NTuple{N,Int}}, + Pair{N, T} where {N,NTuple{N,Int}<:T<:Tuple{Int,Vararg{Int}}}, + !Union{} +) From 9bac7e76f8ae1adfe0d503ecc518012cf3ca7582 Mon Sep 17 00:00:00 2001 From: Bart Janssens Date: Fri, 7 Mar 2025 03:37:50 +0100 Subject: [PATCH 32/74] Fix julia.h for use in MSVC (#57654) Since `__attribute__` is only valid for (mostly?) GCC and Clang, building libraries against Julia with MSVC is currently not working. This change should fix that. (cherry picked from commit 5b06efc362de1e1d8e741526428f1a467db00190) --- src/julia.h | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/julia.h b/src/julia.h index 0dd9ba7027e98..db358804e3006 100644 --- a/src/julia.h +++ b/src/julia.h @@ -727,7 +727,15 @@ static const uint8_t PARTITION_FLAG_DEPRECATED = 0x20; // where calling the function itself will provide a (better) deprecation warning/error. static const uint8_t PARTITION_FLAG_DEPWARN = 0x40; -typedef struct __attribute__((aligned(8))) _jl_binding_partition_t { +#if defined(_COMPILER_MICROSOFT_) +#define JL_ALIGNED_ATTR(alignment) \ + __declspec(align(alignment)) +#else +#define JL_ALIGNED_ATTR(alignment) \ + __attribute__((aligned(alignment))) +#endif + +typedef struct JL_ALIGNED_ATTR(8) _jl_binding_partition_t { JL_DATA_TYPE /* union { * // For ->kind == PARTITION_KIND_GLOBAL From c0bf26a952fdc37006cd2ac6e51f4f666b5114ab Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Fri, 7 Mar 2025 08:17:55 -0500 Subject: [PATCH 33/74] lock: Specialize `OncePerX` more aggressively (#57660) Accidental regression from https://github.com/JuliaLang/julia/pull/57289. Due to the different specialization rules for `Function` types, the `@noinline` bodies were inferring very poorly treating `once` as a `Function` instead of the much more specific `OncePerX{T,F}`. Adds a trimming test case so that we don't regress again. (cherry picked from commit c93dd2db6b9bdd9d10ff9e87ed9c065dce14c092) --- base/lock.jl | 12 ++++++------ test/trimming/hello.jl | 9 ++++++++- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/base/lock.jl b/base/lock.jl index 79a2d1491bc00..06c35f989422d 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -711,10 +711,10 @@ mutable struct OncePerProcess{T, F} <: Function end OncePerProcess{T}(initializer::F) where {T, F} = OncePerProcess{T, F}(initializer) OncePerProcess(initializer) = OncePerProcess{Base.promote_op(initializer), typeof(initializer)}(initializer) -@inline function (once::OncePerProcess{T})() where T +@inline function (once::OncePerProcess{T,F})() where {T,F} state = (@atomic :acquire once.state) if state != PerStateHasrun - (@noinline function init_perprocesss(once, state) + (@noinline function init_perprocesss(once::OncePerProcess{T,F}, state::UInt8) where {T,F} state == PerStateErrored && error("OncePerProcess initializer failed previously") once.allow_compile_time || __precompile__(false) lock(once.lock) @@ -818,14 +818,14 @@ mutable struct OncePerThread{T, F} <: Function end OncePerThread{T}(initializer::F) where {T, F} = OncePerThread{T,F}(initializer) OncePerThread(initializer) = OncePerThread{Base.promote_op(initializer), typeof(initializer)}(initializer) -@inline (once::OncePerThread)() = once[Threads.threadid()] -@inline function getindex(once::OncePerThread, tid::Integer) +@inline (once::OncePerThread{T,F})() where {T,F} = once[Threads.threadid()] +@inline function getindex(once::OncePerThread{T,F}, tid::Integer) where {T,F} tid = Int(tid) ss = @atomic :acquire once.ss xs = @atomic :monotonic once.xs # n.b. length(xs) >= length(ss) if tid <= 0 || tid > length(ss) || (@atomic :acquire ss[tid]) != PerStateHasrun - (@noinline function init_perthread(once, tid) + (@noinline function init_perthread(once::OncePerThread{T,F}, tid::Int) where {T,F} local ss = @atomic :acquire once.ss local xs = @atomic :monotonic once.xs local len = length(ss) @@ -933,6 +933,6 @@ mutable struct OncePerTask{T, F} <: Function OncePerTask{T,F}(initializer::F) where {T, F} = new{T,F}(initializer) OncePerTask(initializer) = new{Base.promote_op(initializer), typeof(initializer)}(initializer) end -@inline function (once::OncePerTask{T})() where {T} +@inline function (once::OncePerTask{T,F})() where {T,F} get!(once.initializer, task_local_storage(), once)::T end diff --git a/test/trimming/hello.jl b/test/trimming/hello.jl index 307bf820f325b..fef25f9e8558f 100644 --- a/test/trimming/hello.jl +++ b/test/trimming/hello.jl @@ -1,6 +1,13 @@ module MyApp + +world::String = "world!" +const str = OncePerProcess{String}() do + return "Hello, " * world +end + Base.@ccallable function main()::Cint - println(Core.stdout, "Hello, world!") + println(Core.stdout, str()) return 0 end + end From 8bfb65885da61fa7408e9bb4c131d02e777c47f7 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Fri, 7 Mar 2025 14:12:41 -0500 Subject: [PATCH 34/74] trimming: improve output on failure (#57657) Remove the unnecessary stack trace and print the error/warning summary on a single line. Before: ```console ... Trim verify finished with 1 warning. Trim verify finished with 57 errors. fatal: error thrown and no exception handler available. ErrorException("verify_typeinf_trim failed") error at ./error.jl:44 verify_typeinf_trim at ./../usr/share/julia/Compiler/src/verifytrim.jl:341 unknown function (ip: 0x7314d73ccaaa) at (unknown file) jl_apply at /home/user/jc/jh-juliac-mwe/julia/src/julia.h:2320 [inlined] jl_f__call_latest at /home/user/jc/jh-juliac-mwe/julia/src/builtins.c:883 #invokelatest#1 at ./essentials.jl:1090 [inlined] invokelatest at ./essentials.jl:1086 [inlined] verify_typeinf_trim at ./../usr/share/julia/Compiler/src/typeinfer.jl:1371 [inlined] typeinf_ext_toplevel at ./../usr/share/julia/Compiler/src/typeinfer.jl:1365 jfptr_typeinf_ext_toplevel_124479 at /home/user/jc/jh-juliac-mwe/julia/usr/lib/julia/sys.so (unknown line) jl_apply at /home/user/jc/jh-juliac-mwe/julia/src/julia.h:2320 [inlined] jl_create_native_impl at /home/user/jc/jh-juliac-mwe/julia/src/aotcompile.cpp:667 jl_precompile_trimmed at /home/user/jc/jh-juliac-mwe/julia/src/precompile_utils.c:385 [inlined] ijl_create_system_image at /home/user/jc/jh-juliac-mwe/julia/src/staticdata.c:3419 ijl_write_compiler_output at /home/user/jc/jh-juliac-mwe/julia/src/precompile.c:155 ijl_atexit_hook at /home/user/jc/jh-juliac-mwe/julia/src/init.c:278 jl_repl_entrypoint at /home/user/jc/jh-juliac-mwe/julia/src/jlapi.c:1125 main at /home/user/jc/jh-juliac-mwe/julia/cli/loader_exe.c:58 unknown function (ip: 0x7314e1029d8f) at /lib/x86_64-linux-gnu/libc.so.6 __libc_start_main at /lib/x86_64-linux-gnu/libc.so.6 (unknown line) _start at /home/user/jc/jh-juliac-mwe/julia/usr/bin/julia (unknown line) Failed to compile ./test/trimming/hello.jl ``` After: ```console ... Trim verify finished with 57 errors, 1 warning. Failed to compile ./test/trimming/hello.jl ``` (cherry picked from commit 6acc0d7369bb05f5e92452cb2d8fe6be78f49db4) --- Compiler/src/verifytrim.jl | 11 ++++++---- base/boot.jl | 1 + src/jl_exported_data.inc | 9 +++++---- src/jltypes.c | 41 ++++++++++++++++++++------------------ src/julia.h | 1 + src/precompile_utils.c | 13 +++++++++++- src/staticdata.c | 3 ++- 7 files changed, 50 insertions(+), 29 deletions(-) diff --git a/Compiler/src/verifytrim.jl b/Compiler/src/verifytrim.jl index 36f9413a8c1a7..5a80082c63330 100644 --- a/Compiler/src/verifytrim.jl +++ b/Compiler/src/verifytrim.jl @@ -327,16 +327,19 @@ function verify_typeinf_trim(io::IO, codeinfos::Vector{Any}, onlywarn::Bool) end let severity = 0 - if counts[2] > 0 - print("Trim verify finished with ", counts[2], counts[2] == 1 ? " warning.\n\n" : " warnings.\n\n") + if counts[1] > 0 || counts[2] > 0 + print("Trim verify finished with ") + print(counts[1], counts[1] == 1 ? " error" : " errors") + print(", ") + print(counts[2], counts[2] == 1 ? " warning" : " warnings") + print(".\n") severity = 2 end if counts[1] > 0 - print("Trim verify finished with ", counts[1], counts[1] == 1 ? " error.\n\n" : " errors.\n\n") severity = 1 end # messages classified as errors are fatal, warnings are not - 0 < severity <= 1 && !onlywarn && error("verify_typeinf_trim failed") + 0 < severity <= 1 && !onlywarn && throw(Core.TrimFailure()) end nothing end diff --git a/base/boot.jl b/base/boot.jl index b1abe6a65041f..8cd032817cebe 100644 --- a/base/boot.jl +++ b/base/boot.jl @@ -475,6 +475,7 @@ struct ABIOverride end struct PrecompilableError <: Exception end +struct TrimFailure <: Exception end String(s::String) = s # no constructor yet diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 62acce6ce1d65..df3b9c121837c 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -2,6 +2,7 @@ // Pointers that are exposed through the public libjulia #define JL_EXPORTED_DATA_POINTERS(XX) \ + XX(jl_abioverride_type) \ XX(jl_abstractarray_type) \ XX(jl_abstractstring_type) \ XX(jl_addrspace_type) \ @@ -65,6 +66,8 @@ XX(jl_interrupt_exception) \ XX(jl_intrinsic_type) \ XX(jl_kwcall_func) \ + XX(jl_libdl_module) \ + XX(jl_libdl_dlopen_func) \ XX(jl_lineinfonode_type) \ XX(jl_linenumbernode_type) \ XX(jl_llvmpointer_type) \ @@ -108,6 +111,7 @@ XX(jl_pinode_type) \ XX(jl_pointer_type) \ XX(jl_pointer_typename) \ + XX(jl_precompilable_error) \ XX(jl_quotenode_type) \ XX(jl_readonlymemory_exception) \ XX(jl_ref_type) \ @@ -116,12 +120,12 @@ XX(jl_simplevector_type) \ XX(jl_slotnumber_type) \ XX(jl_ssavalue_type) \ - XX(jl_abioverride_type) \ XX(jl_stackovf_exception) \ XX(jl_string_type) \ XX(jl_symbol_type) \ XX(jl_task_type) \ XX(jl_top_module) \ + XX(jl_trimfailure_type) \ XX(jl_true) \ XX(jl_tuple_typename) \ XX(jl_tvar_type) \ @@ -149,9 +153,6 @@ XX(jl_voidpointer_type) \ XX(jl_void_type) \ XX(jl_weakref_type) \ - XX(jl_libdl_module) \ - XX(jl_libdl_dlopen_func) \ - XX(jl_precompilable_error) \ // Data symbols that are defined inside the public libjulia #define JL_EXPORTED_DATA_SYMBOLS(XX) \ diff --git a/src/jltypes.c b/src/jltypes.c index e6d48b6f489b2..d2dbfbc4fe439 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3934,27 +3934,30 @@ void post_boot_hooks(void) jl_int32_type->super = jl_signed_type; jl_int64_type->super = jl_signed_type; - jl_errorexception_type = (jl_datatype_t*)core("ErrorException"); - jl_stackovf_exception = jl_new_struct_uninit((jl_datatype_t*)core("StackOverflowError")); - jl_diverror_exception = jl_new_struct_uninit((jl_datatype_t*)core("DivideError")); - jl_undefref_exception = jl_new_struct_uninit((jl_datatype_t*)core("UndefRefError")); - jl_undefvarerror_type = (jl_datatype_t*)core("UndefVarError"); - jl_fielderror_type = (jl_datatype_t*)core("FieldError"); - jl_atomicerror_type = (jl_datatype_t*)core("ConcurrencyViolationError"); - jl_interrupt_exception = jl_new_struct_uninit((jl_datatype_t*)core("InterruptException")); - jl_boundserror_type = (jl_datatype_t*)core("BoundsError"); - jl_memory_exception = jl_new_struct_uninit((jl_datatype_t*)core("OutOfMemoryError")); + jl_stackovf_exception = jl_new_struct_uninit((jl_datatype_t*)core("StackOverflowError")); + jl_diverror_exception = jl_new_struct_uninit((jl_datatype_t*)core("DivideError")); + jl_undefref_exception = jl_new_struct_uninit((jl_datatype_t*)core("UndefRefError")); + jl_interrupt_exception = jl_new_struct_uninit((jl_datatype_t*)core("InterruptException")); + jl_memory_exception = jl_new_struct_uninit((jl_datatype_t*)core("OutOfMemoryError")); jl_readonlymemory_exception = jl_new_struct_uninit((jl_datatype_t*)core("ReadOnlyMemoryError")); - jl_typeerror_type = (jl_datatype_t*)core("TypeError"); - jl_argumenterror_type = (jl_datatype_t*)core("ArgumentError"); - jl_methoderror_type = (jl_datatype_t*)core("MethodError"); - jl_loaderror_type = (jl_datatype_t*)core("LoadError"); - jl_initerror_type = (jl_datatype_t*)core("InitError"); + jl_precompilable_error = jl_new_struct_uninit((jl_datatype_t*)core("PrecompilableError")); + + jl_errorexception_type = (jl_datatype_t*)core("ErrorException"); + jl_undefvarerror_type = (jl_datatype_t*)core("UndefVarError"); + jl_fielderror_type = (jl_datatype_t*)core("FieldError"); + jl_atomicerror_type = (jl_datatype_t*)core("ConcurrencyViolationError"); + jl_boundserror_type = (jl_datatype_t*)core("BoundsError"); + jl_typeerror_type = (jl_datatype_t*)core("TypeError"); + jl_argumenterror_type = (jl_datatype_t*)core("ArgumentError"); + jl_methoderror_type = (jl_datatype_t*)core("MethodError"); + jl_loaderror_type = (jl_datatype_t*)core("LoadError"); + jl_initerror_type = (jl_datatype_t*)core("InitError"); jl_missingcodeerror_type = (jl_datatype_t*)core("MissingCodeError"); - jl_precompilable_error = jl_new_struct_uninit((jl_datatype_t*)core("PrecompilableError")); - jl_pair_type = core("Pair"); - jl_kwcall_func = core("kwcall"); - jl_kwcall_mt = ((jl_datatype_t*)jl_typeof(jl_kwcall_func))->name->mt; + jl_trimfailure_type = (jl_datatype_t*)core("TrimFailure"); + + jl_pair_type = core("Pair"); + jl_kwcall_func = core("kwcall"); + jl_kwcall_mt = ((jl_datatype_t*)jl_typeof(jl_kwcall_func))->name->mt; jl_atomic_store_relaxed(&jl_kwcall_mt->max_args, 0); jl_weakref_type = (jl_datatype_t*)core("WeakRef"); diff --git a/src/julia.h b/src/julia.h index db358804e3006..306f6a914eaac 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1020,6 +1020,7 @@ extern JL_DLLIMPORT jl_datatype_t *jl_undefvarerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_fielderror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_atomicerror_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_missingcodeerror_type JL_GLOBALLY_ROOTED; +extern JL_DLLIMPORT jl_datatype_t *jl_trimfailure_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_lineinfonode_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_datatype_t *jl_abioverride_type JL_GLOBALLY_ROOTED; extern JL_DLLIMPORT jl_value_t *jl_stackovf_exception JL_GLOBALLY_ROOTED; diff --git a/src/precompile_utils.c b/src/precompile_utils.c index 8906b3eb586d3..281dbe0163586 100644 --- a/src/precompile_utils.c +++ b/src/precompile_utils.c @@ -382,7 +382,18 @@ static void *jl_precompile_trimmed(size_t world) jl_array_ptr_1d_push(m, ccallable); } - void *native_code = jl_create_native(m, NULL, jl_options.trim, 0, world); + void *native_code = NULL; + JL_TRY { + native_code = jl_create_native(m, NULL, jl_options.trim, 0, world); + } JL_CATCH { + jl_value_t *exc = jl_current_exception(jl_current_task); + if (!jl_isa(exc, (jl_value_t*)jl_trimfailure_type)) + jl_rethrow(); // unexpected exception, expose the stacktrace + + // The verification check failed. The error message should already have + // been printed, so give up here and exit (w/o a stack trace). + exit(1); + } JL_GC_POP(); return native_code; } diff --git a/src/staticdata.c b/src/staticdata.c index 724019fbe1559..b450cf2561107 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -102,7 +102,7 @@ extern "C" { // TODO: put WeakRefs on the weak_refs list during deserialization // TODO: handle finalizers -#define NUM_TAGS 196 +#define NUM_TAGS 197 // An array of references that need to be restored from the sysimg // This is a manually constructed dual of the gvars array, which would be produced by codegen for Julia code, for C. @@ -250,6 +250,7 @@ jl_value_t **const*const get_tags(void) { INSERT_TAG(jl_atomicerror_type); INSERT_TAG(jl_missingcodeerror_type); INSERT_TAG(jl_precompilable_error); + INSERT_TAG(jl_trimfailure_type); // other special values INSERT_TAG(jl_emptysvec); From 36986a03ba217600aeb90b68b33241205877109b Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:43:53 -0500 Subject: [PATCH 35/74] typeinfer: Add `force_enable_inference` to override Module-level inference settings (#57653) Allows `--trim` to infer all the way through `__init__()` in JLLWrappers.jl despite the `@compiler_options` disabling inference (https://github.com/JuliaPackaging/JLLWrappers.jl/blob/ecaba50a4462209714f0979667c64c1bf28ee892/src/toplevel_generators.jl#L55) Required for #57587. (cherry picked from commit 7ef92c8876a82fc0a649a1d964a78bbf0782a704) --- Compiler/src/typeinfer.jl | 27 ++++++++++++++++----------- Compiler/src/types.jl | 25 ++++++++++++++++++++----- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index fb44bc7cc8989..385cf0ef10894 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -862,7 +862,7 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize end end end - if ccall(:jl_get_module_infer, Cint, (Any,), method.module) == 0 + if !InferenceParams(interp).force_enable_inference && ccall(:jl_get_module_infer, Cint, (Any,), method.module) == 0 add_remark!(interp, caller, "[typeinf_edge] Inference is disabled for the target module") return Future(MethodCallResult(interp, caller, method, Any, Any, Effects(), nothing, edgecycle, edgelimited)) end @@ -1144,15 +1144,17 @@ function typeinf_ext(interp::AbstractInterpreter, mi::MethodInstance, source_mod end end end - if isa(def, Method) && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 - src = retrieve_code_info(mi, get_inference_world(interp)) - if src isa CodeInfo - finish!(interp, mi, ci, src) - else - engine_reject(interp, ci) + if !InferenceParams(interp).force_enable_inference + if isa(def, Method) && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 + src = retrieve_code_info(mi, get_inference_world(interp)) + if src isa CodeInfo + finish!(interp, mi, ci, src) + else + engine_reject(interp, ci) + end + ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) + return ci end - ccall(:jl_typeinf_timing_end, Cvoid, (UInt64,), start_time) - return ci end result = InferenceResult(mi, typeinf_lattice(interp)) result.ci = ci @@ -1284,7 +1286,10 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m # first compute the ABIs of everything latest = true # whether this_world == world_counter() for this_world in reverse(sort!(worlds)) - interp = NativeInterpreter(this_world) + interp = NativeInterpreter( + this_world; + inf_params = InferenceParams(; force_enable_inference = trim_mode != TRIM_NO) + ) for i = 1:length(methods) # each item in this list is either a MethodInstance indicating something # to compile, or an svec(rettype, sig) describing a C-callable alias to create. @@ -1325,7 +1330,7 @@ function typeinf_ext_toplevel(methods::Vector{Any}, worlds::Vector{UInt}, trim_m src = codeinfo_for_const(interp, mi, callee.rettype_const) elseif haskey(interp.codegen, callee) src = interp.codegen[callee] - elseif isa(def, Method) && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 && trim_mode == TRIM_NO + elseif isa(def, Method) && !InferenceParams(interp).force_enable_inference && ccall(:jl_get_module_infer, Cint, (Any,), def.module) == 0 src = retrieve_code_info(mi, get_inference_world(interp)) else # TODO: typeinf_code could return something with different edges/ages/owner/abi (needing an update to callee), which we don't handle here diff --git a/Compiler/src/types.jl b/Compiler/src/types.jl index 306728a8a21e9..eb05ba2b8daa6 100644 --- a/Compiler/src/types.jl +++ b/Compiler/src/types.jl @@ -187,6 +187,10 @@ Parameters that control abstract interpretation-based type inference operation. it will `throw`). Defaults to `false` since this assumption does not hold in Julia's semantics for native code execution. --- +- `inf_params.force_enable_inference::Bool = false`\\ + If `true`, inference will be performed on functions regardless of whether it was disabled + at the module level via `Base.Experimental.@compiler_options`. +--- """ struct InferenceParams max_methods::Int @@ -198,6 +202,7 @@ struct InferenceParams aggressive_constant_propagation::Bool assume_bindings_static::Bool ignore_recursion_hardlimit::Bool + force_enable_inference::Bool function InferenceParams( max_methods::Int, @@ -208,7 +213,9 @@ struct InferenceParams ipo_constant_propagation::Bool, aggressive_constant_propagation::Bool, assume_bindings_static::Bool, - ignore_recursion_hardlimit::Bool) + ignore_recursion_hardlimit::Bool, + force_enable_inference::Bool, + ) return new( max_methods, max_union_splitting, @@ -218,7 +225,9 @@ struct InferenceParams ipo_constant_propagation, aggressive_constant_propagation, assume_bindings_static, - ignore_recursion_hardlimit) + ignore_recursion_hardlimit, + force_enable_inference, + ) end end function InferenceParams( @@ -231,7 +240,9 @@ function InferenceParams( #=ipo_constant_propagation::Bool=# true, #=aggressive_constant_propagation::Bool=# false, #=assume_bindings_static::Bool=# false, - #=ignore_recursion_hardlimit::Bool=# false); + #=ignore_recursion_hardlimit::Bool=# false, + #=force_enable_inference::Bool=# false + ); max_methods::Int = params.max_methods, max_union_splitting::Int = params.max_union_splitting, max_apply_union_enum::Int = params.max_apply_union_enum, @@ -240,7 +251,9 @@ function InferenceParams( ipo_constant_propagation::Bool = params.ipo_constant_propagation, aggressive_constant_propagation::Bool = params.aggressive_constant_propagation, assume_bindings_static::Bool = params.assume_bindings_static, - ignore_recursion_hardlimit::Bool = params.ignore_recursion_hardlimit) + ignore_recursion_hardlimit::Bool = params.ignore_recursion_hardlimit, + force_enable_inference::Bool = params.force_enable_inference, +) return InferenceParams( max_methods, max_union_splitting, @@ -250,7 +263,9 @@ function InferenceParams( ipo_constant_propagation, aggressive_constant_propagation, assume_bindings_static, - ignore_recursion_hardlimit) + ignore_recursion_hardlimit, + force_enable_inference, + ) end """ From 0e93f123908994ce7d23747e7e7f65dd67ce853a Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 10 Mar 2025 14:25:21 +0100 Subject: [PATCH 36/74] Compiler: `abstract_apply`: declare type of two closure captures (#57684) These two local variables were, unlike some other ones, missing type declarations. The type declarations are helpful becase the variables get captured and assigned to in the closure `infercalls`, resulting in bad type inference without the type declarations. I don't think there's any drawback to adding the type declarations: * the declared type is concrete * conversion shouldn't ever happen, as only values of the same type get assigned to these variables This change decreases the number of invalidations on loading package TypeDomainNaturalNumbers v6.1.0 (without first loading the REPL) from 981 to 968. The comparison is on top of eba2a337f914462b47a26fb30939155beab72ace. (cherry picked from commit b1fffb06c8ea49ac8298999f057e456fb500ab01) --- Compiler/src/abstractinterpretation.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index 212169c107942..b29b65b737382 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -1736,8 +1736,8 @@ function abstract_apply(interp::AbstractInterpreter, argtypes::Vector{Any}, si:: retinfos = ApplyCallInfo[] retinfo = UnionSplitApplyCallInfo(retinfos) exctype = Union{} - ctypes´ = Vector{Any}[] - infos´ = Vector{MaybeAbstractIterationInfo}[] + ctypes´::Vector{Vector{Any}} = Vector{Any}[] + infos´::Vector{Vector{MaybeAbstractIterationInfo}} = Vector{MaybeAbstractIterationInfo}[] local ti, argtypesi local ctfuture::Future{AbstractIterationResult} local callfuture::Future{CallMeta} From c04db34ff6bfd75d1b6e6ca89d8c063f08641648 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Tue, 11 Mar 2025 12:17:35 +0100 Subject: [PATCH 37/74] `Base`: `append!`, `resize!`: convert length to `Int` before passing it on (#57585) Reduces the number of invalidations from 512 to 505 on running this code: ```julia struct I <: Integer end Base.Int(::I) = 7 ``` (cherry picked from commit a97137ee079de308599c72e1ed04154dbd3ffc50) --- base/array.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/base/array.jl b/base/array.jl index aafcfc182124b..f134ad2bc9ea5 100644 --- a/base/array.jl +++ b/base/array.jl @@ -1348,7 +1348,7 @@ function append! end function append!(a::Vector{T}, items::Union{AbstractVector{<:T},Tuple}) where T items isa Tuple && (items = map(x -> convert(T, x), items)) - n = length(items) + n = Int(length(items))::Int _growend!(a, n) copyto!(a, length(a)-n+1, items, firstindex(items), n) return a @@ -1472,7 +1472,8 @@ julia> a[1:6] 1 ``` """ -function resize!(a::Vector, nl::Integer) +function resize!(a::Vector, nl_::Integer) + nl = Int(nl_)::Int l = length(a) if nl > l _growend!(a, nl-l) From d8af2ed6a02294e351ab7afcf86b16fa609b35dd Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Tue, 11 Mar 2025 17:03:43 +0100 Subject: [PATCH 38/74] Compiler/ssair/passes: `_lift_svec_ref`: improve type stability (#57633) Making the closure capture a `SimpleVector` instead of a type should improve type stability. Also deduplicated some common subexpressions while at it. (cherry picked from commit 885b1cd875f101f227b345f681cc36879124d80d) --- Compiler/src/ssair/passes.jl | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/Compiler/src/ssair/passes.jl b/Compiler/src/ssair/passes.jl index c9b3d5515caa3..14fc0ab20913c 100644 --- a/Compiler/src/ssair/passes.jl +++ b/Compiler/src/ssair/passes.jl @@ -1027,17 +1027,19 @@ end sig = sig.body isa(sig, DataType) || return nothing sig.name === Tuple.name || return nothing - length(sig.parameters) >= 1 || return nothing + sig_parameters = sig.parameters::SimpleVector + length_sig_parameters = length(sig_parameters) + length_sig_parameters >= 1 || return nothing - i = let sig=sig - findfirst(j::Int->has_typevar(sig.parameters[j], tvar), 1:length(sig.parameters)) + function has_typevar_closure(j::Int) + has_typevar(sig_parameters[j], tvar) end + + i = findfirst(has_typevar_closure, 1:length_sig_parameters) i === nothing && return nothing - let sig=sig - any(j::Int->has_typevar(sig.parameters[j], tvar), i+1:length(sig.parameters)) - end && return nothing + any(has_typevar_closure, i+1:length_sig_parameters) && return nothing - arg = sig.parameters[i] + arg = sig_parameters[i] rarg = def.args[2 + i] isa(rarg, SSAValue) || return nothing From 1ad29822bfde161328db370d73593b3b4ea4b14e Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 11 Mar 2025 15:20:11 -0400 Subject: [PATCH 39/74] Libdl: Improve inference for `string(::LazyLibraryPath)` (#57721) Make this function `--trim` compatible. (cherry picked from commit 8edc2b35fca93a002606219bc0bb4628c5b986fa) --- base/libdl.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/libdl.jl b/base/libdl.jl index 199d847572ca4..0b603d944100b 100644 --- a/base/libdl.jl +++ b/base/libdl.jl @@ -334,7 +334,7 @@ struct LazyLibraryPath LazyLibraryPath(pieces::Vector) = new(pieces) end LazyLibraryPath(args...) = LazyLibraryPath(collect(args)) -Base.string(llp::LazyLibraryPath) = joinpath(string.(llp.pieces)...)::String +Base.string(llp::LazyLibraryPath) = joinpath(String[string(p) for p in llp.pieces]) Base.cconvert(::Type{Cstring}, llp::LazyLibraryPath) = Base.cconvert(Cstring, string(llp)) # Define `print` so that we can wrap this in a `LazyString` Base.print(io::IO, llp::LazyLibraryPath) = print(io, string(llp)) From 7e0392d56c7ccc558719b038377b1440395b1b7e Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Tue, 11 Mar 2025 15:42:43 -0400 Subject: [PATCH 40/74] fix alignment computation for nested objects (#57722) The alignment of a nested object (in C layouts) is not affected by the alignment of its parent container when computing a field offset (as if it will be allocated at address 0). This can be strongly counter-intuitive (as it implies it should add padding where it does not seem to provide value to the user), but this is required to match the C standard. It also permits users to write custom allocators which happen to provide alignment in excess of that which codegen may assume is guaranteed, and get the behavioral characteristics they intended to specify (without resorting to computing explicit padding). Addresses https://github.com/JuliaLang/julia/issues/57713#issuecomment-2714111074 (cherry picked from commit c9008ffab74ce2ffc13e3e1d18b0b83277487133) --- doc/src/manual/calling-c-and-fortran-code.md | 24 ++++++++++++------- src/cgutils.cpp | 13 ++++------ src/datatype.c | 25 ++++++++++++++------ test/core.jl | 7 ++++++ 4 files changed, 45 insertions(+), 24 deletions(-) diff --git a/doc/src/manual/calling-c-and-fortran-code.md b/doc/src/manual/calling-c-and-fortran-code.md index d198c796a2e0b..aa317468b0f75 100644 --- a/doc/src/manual/calling-c-and-fortran-code.md +++ b/doc/src/manual/calling-c-and-fortran-code.md @@ -547,15 +547,14 @@ is not valid, since the type layout of `T` is not known statically. ### SIMD Values -Note: This feature is currently implemented on 64-bit x86 and AArch64 platforms only. - If a C/C++ routine has an argument or return value that is a native SIMD type, the corresponding Julia type is a homogeneous tuple of `VecElement` that naturally maps to the SIMD type. Specifically: -> * The tuple must be the same size as the SIMD type. For example, a tuple representing an `__m128` -> on x86 must have a size of 16 bytes. -> * The element type of the tuple must be an instance of `VecElement{T}` where `T` is a primitive type that -> is 1, 2, 4 or 8 bytes. +> * The tuple must be the same size and elements as the SIMD type. For example, a tuple +> representing an `__m128` on x86 must have a size of 16 bytes and Float32 elements. +> * The element type of the tuple must be an instance of `VecElement{T}` where `T` is a +> primitive type with a power-of-two number of bytes (e.g. 1, 2, 4, 8, 16, etc) such as +> Int8 or Float64. For instance, consider this C routine that uses AVX intrinsics: @@ -628,6 +627,10 @@ For translating a C argument list to Julia: * `T`, where `T` is a concrete Julia type * argument value will be copied (passed by value) + * `vector T` (or `__attribute__ vector_size`, or a typedef such as `__m128`) + + * `NTuple{N, VecElement{T}}`, where `T` is a primitive Julia type of the correct size + and N is the number of elements in the vector (equal to `vector_size / sizeof T`). * `void*` * depends on how this parameter is used, first translate this to the intended pointer type, then @@ -674,13 +677,16 @@ For translating a C return type to Julia: * `T`, where `T` is one of the primitive types: `char`, `int`, `long`, `short`, `float`, `double`, `complex`, `enum` or any of their `typedef` equivalents - * `T`, where `T` is an equivalent Julia Bits Type (per the table above) - * if `T` is an `enum`, the argument type should be equivalent to `Cint` or `Cuint` + * same as C argument list * argument value will be copied (returned by-value) * `struct T` (including typedef to a struct) - * `T`, where `T` is a concrete Julia Type + * same as C argument list * argument value will be copied (returned by-value) + + * `vector T` + + * same as C argument list * `void*` * depends on how this parameter is used, first translate this to the intended pointer type, then diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 56c5dd48e649f..1e8a1e475e2ee 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -3103,11 +3103,7 @@ static jl_cgval_t emit_getfield_knownidx(jl_codectx_t &ctx, const jl_cgval_t &st else if (strct.ispointer()) { auto tbaa = best_field_tbaa(ctx, strct, jt, idx, byte_offset); Value *staddr = data_pointer(ctx, strct); - Value *addr; - if (jl_is_vecelement_type((jl_value_t*)jt) || byte_offset == 0) - addr = staddr; // VecElement types are unwrapped in LLVM. - else - addr = emit_ptrgep(ctx, staddr, byte_offset); + Value *addr = (byte_offset == 0 ? staddr : emit_ptrgep(ctx, staddr, byte_offset)); if (addr != staddr) setNameWithField(ctx.emission_context, addr, get_objname, jt, idx, Twine("_ptr")); if (jl_field_isptr(jt, idx)) { @@ -3571,7 +3567,7 @@ static void union_alloca_type(jl_uniontype_t *ut, [&](unsigned idx, jl_datatype_t *jt) { if (!jl_is_datatype_singleton(jt)) { size_t nb1 = jl_datatype_size(jt); - size_t align1 = jl_datatype_align(jt); + size_t align1 = julia_alignment((jl_value_t*)jt); if (nb1 > nbytes) nbytes = nb1; if (align1 > align) @@ -4117,10 +4113,11 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg // choose whether we should perform the initialization with the struct as a IR value // or instead initialize the stack buffer with stores (the later is nearly always better) + // although we do the former if it is a vector or could be a vector element auto tracked = split_value_size(sty); assert(CountTrackedPointers(lt).count == tracked.second); bool init_as_value = false; - if (lt->isVectorTy() || jl_is_vecelement_type(ty)) { // maybe also check the size ? + if (lt->isVectorTy() || jl_special_vector_alignment(1, ty) != 0) { init_as_value = true; } @@ -4327,7 +4324,7 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg if (strct) { jl_aliasinfo_t ai = jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack); promotion_point = ai.decorateInst(ctx.builder.CreateMemSet(strct, ConstantInt::get(getInt8Ty(ctx.builder.getContext()), 0), - jl_datatype_size(ty), MaybeAlign(jl_datatype_align(ty)))); + jl_datatype_size(ty), Align(julia_alignment(ty)))); } ctx.builder.restoreIP(savedIP); } diff --git a/src/datatype.c b/src/datatype.c index fd25cca503676..9c2360c7eeccb 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -300,9 +300,10 @@ static jl_datatype_layout_t *jl_get_layout(uint32_t sz, } // Determine if homogeneous tuple with fields of type t will have -// a special alignment beyond normal Julia rules. +// a special alignment and vector-ABI beyond normal rules for aggregates. // Return special alignment if one exists, 0 if normal alignment rules hold. // A non-zero result *must* match the LLVM rules for a vector type . +// Matching the compiler's `__attribute__ vector_size` behavior. // For sake of Ahead-Of-Time (AOT) compilation, this routine has to work // without LLVM being available. unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) @@ -317,8 +318,12 @@ unsigned jl_special_vector_alignment(size_t nfields, jl_value_t *t) // motivating use case comes up for Julia, we reject pointers. return 0; size_t elsz = jl_datatype_size(ty); - if (elsz != 1 && elsz != 2 && elsz != 4 && elsz != 8) - // Only handle power-of-two-sized elements (for now) + if (next_power_of_two(elsz) != elsz) + // Only handle power-of-two-sized elements (for now), since other + // lengths may be packed into very complicated arrangements (llvm pads + // extra bits on most platforms when computing alignment but not when + // computing type size, but adds no extra bytes for each element, so + // their effect on offsets are never what you may naturally expect). return 0; size_t size = nfields * elsz; // Use natural alignment for this vector: this matches LLVM and clang. @@ -723,9 +728,9 @@ void jl_compute_field_offsets(jl_datatype_t *st) } else { fsz = sizeof(void*); - if (fsz > MAX_ALIGN) - fsz = MAX_ALIGN; al = fsz; + if (al > MAX_ALIGN) + al = MAX_ALIGN; desc[i].isptr = 1; zeroinit = 1; npointers++; @@ -769,8 +774,6 @@ void jl_compute_field_offsets(jl_datatype_t *st) if (al > alignm) alignm = al; } - if (alignm > MAX_ALIGN) - alignm = MAX_ALIGN; // We cannot guarantee alignments over 16 bytes because that's what our heap is aligned as if (LLT_ALIGN(sz, alignm) > sz) { haspadding = 1; sz = LLT_ALIGN(sz, alignm); @@ -939,6 +942,14 @@ JL_DLLEXPORT jl_datatype_t *jl_new_primitivetype(jl_value_t *name, jl_module_t * uint32_t nbytes = (nbits + 7) / 8; uint32_t alignm = next_power_of_two(nbytes); # if defined(_CPU_X86_) && !defined(_OS_WINDOWS_) + // datalayout strings are often weird: on 64-bit they usually follow fairly simple rules, + // but on x86 32 bit platforms, sometimes 5 to 8 byte types are + // 32-bit aligned even though the MAX_ALIGN (for types 9+ bytes) is 16 + // (except for f80 which is align 4 on Mingw, Linux, and BSDs--but align 16 on MSVC and Darwin) + // https://llvm.org/doxygen/ARMTargetMachine_8cpp.html#adb29b487708f0dc2a940345b68649270 + // https://llvm.org/doxygen/AArch64TargetMachine_8cpp.html#a003a58caf135efbf7273c5ed84e700d7 + // https://llvm.org/doxygen/X86TargetMachine_8cpp.html#aefdbcd6131ef195da070cef7fdaf0532 + // 32-bit alignment is weird if (alignm == 8) alignm = 4; # endif diff --git a/test/core.jl b/test/core.jl index 4502d82e1f529..036ca63eb70b4 100644 --- a/test/core.jl +++ b/test/core.jl @@ -5773,6 +5773,13 @@ let ni128 = sizeof(FP128test) ÷ sizeof(Int), @test reinterpret(UInt128, arr[2].fp) == expected end +# make sure VecElement Tuple has the C alignment and ABI for supported types +primitive type Int24 24 end +@test Base.datatype_alignment(NTuple{10,VecElement{Int16}}) == 32 +@test Base.datatype_alignment(NTuple{10,VecElement{Int24}}) == 4 +@test Base.datatype_alignment(NTuple{10,VecElement{Int64}}) == 128 +@test Base.datatype_alignment(NTuple{10,VecElement{Int128}}) == 256 + # issue #21516 struct T21516 x::Vector{Float64} From 21d447244713a28674e3dc490bab6262696479ff Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Tue, 11 Mar 2025 22:08:03 -0400 Subject: [PATCH 41/74] Libdl: Store path contents as `Tuple` instead of `Vector` (#57734) This makes `LazyLibraryPath` eligible for const-prop, which is important so that we can inline/infer `string(::LazyLibraryPath)` at compile-time if the argument is a (global) `const`. Otherwise, this code is not infer-able without something like TypedCallable to make the interface requirements explicit. (cherry picked from commit 3f4eda64feb0267ae71f603e9ce80fc05966cdc5) --- base/libdl.jl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/base/libdl.jl b/base/libdl.jl index 0b603d944100b..de8c6a7a597c5 100644 --- a/base/libdl.jl +++ b/base/libdl.jl @@ -330,10 +330,9 @@ libfoo = LazyLibrary(LazyLibraryPath(prefix, "lib/libfoo.so.1.2.3")) ``` """ struct LazyLibraryPath - pieces::Vector - LazyLibraryPath(pieces::Vector) = new(pieces) + pieces::Tuple{Vararg{Any}} + LazyLibraryPath(pieces...) = new(pieces) end -LazyLibraryPath(args...) = LazyLibraryPath(collect(args)) Base.string(llp::LazyLibraryPath) = joinpath(String[string(p) for p in llp.pieces]) Base.cconvert(::Type{Cstring}, llp::LazyLibraryPath) = Base.cconvert(Cstring, string(llp)) # Define `print` so that we can wrap this in a `LazyString` From c2398b40bbac89258a58b7db86cb90b0a54a0346 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Wed, 12 Mar 2025 08:50:24 +0100 Subject: [PATCH 42/74] type assert `Core.SimpleVector` in `typejoin_union_tuple` (#57631) Should get rid of some invalidation on running user code. (cherry picked from commit d96a6e63d3eb1f44b4258233ce1118cca77d1249) --- base/promotion.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/promotion.jl b/base/promotion.jl index 72257f8ba5a3d..719cd2dc32b61 100644 --- a/base/promotion.jl +++ b/base/promotion.jl @@ -199,7 +199,7 @@ end function typejoin_union_tuple(T::DataType) @_foldable_meta - p = T.parameters + p = T.parameters::Core.SimpleVector lr = length(p) if lr == 0 return Tuple{} From c8fe2f53b701410bd9d607dabda3deed9a5f4c5f Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Wed, 12 Mar 2025 08:55:12 +0100 Subject: [PATCH 43/74] errorshow: typeassert `::SimpleVector` in `_collapse_repeated_frames` (#57691) Reduces the invalidation count on loading the package TypeDomainNaturalNumbers v6.1.0 from 980 to 968. (cherry picked from commit 502612efb6849380fec7469437625d099c396d16) --- base/errorshow.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index d4b9b3666fbb7..2ac1a869a8994 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -918,7 +918,7 @@ function _collapse_repeated_frames(trace) m, last_m = StackTraces.frame_method_or_module(frame), StackTraces.frame_method_or_module(last_frame) if m isa Method && last_m isa Method - params, last_params = Base.unwrap_unionall(m.sig).parameters, Base.unwrap_unionall(last_m.sig).parameters + params, last_params = Base.unwrap_unionall(m.sig).parameters::SimpleVector, Base.unwrap_unionall(last_m.sig).parameters::SimpleVector if last_m.nkw != 0 pos_sig_params = last_params[(last_m.nkw+2):end] issame = true From 1692eb74b8e4f3d7ab5505d85d4d1fda1a76c3c9 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 12 Mar 2025 14:40:25 +0100 Subject: [PATCH 44/74] Avoid requiring the REPL to be loaded to show error hints for undefined variables (#57576) For some reason, the nice error hints for undefined variables are put in the REPL module, requiring it to be loaded to get good error messages. There are many cases where code runs outside of the REPL but where good error messages are still useful (PkgEval, `Pkg.test`, Pluto, IJulia). As an example, this is what I got when running `Pkg.test` for a package with a regression: ``` Testing Running tests... ERROR: LoadError: UndefVarError: `decompress_symmetric` not defined in `Main` Stacktrace: [1] top-level scope @ ~/JuliaPkgs/NonconvexSemidefinite.jl/test/runtests.jl:6 [2] macro expansion ``` But if I paste the code into the REPL ``` julia> decompress_symmetric ERROR: UndefVarError: `decompress_symmetric` not defined in `Main` Hint: It looks like two or more modules export different bindings with this name, resulting in ambiguity. Try explicitly importing it from a particular module, or qualifying the name with the module it should come from. Hint: a global variable of this name also exists in NonconvexCore. Hint: a global variable of this name also exists in NonconvexSemidefinite. Hint: a global variable of this name also exists in NonconvexIpopt. ``` which makes me immediately understand what is going on. This just moves the code for this error hint to live beside all the other hints, I see no reason for keeping this special. (cherry picked from commit 1431bec1bcd205f181ca2a3f1c314247b64076df) --- base/errorshow.jl | 90 ++++++++++++++++++++++++++++++++++++++++ stdlib/REPL/src/REPL.jl | 88 +-------------------------------------- stdlib/REPL/test/repl.jl | 79 ----------------------------------- test/errorshow.jl | 62 ++++++++++++++++++++++++++- 4 files changed, 153 insertions(+), 166 deletions(-) diff --git a/base/errorshow.jl b/base/errorshow.jl index 2ac1a869a8994..0fb6d0782e85f 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -1141,6 +1141,96 @@ end Experimental.register_error_hint(fielderror_listfields_hint_handler, FieldError) +function UndefVarError_hint(io::IO, ex::UndefVarError) + var = ex.var + if isdefined(ex, :scope) + scope = ex.scope + if scope isa Module + bpart = Base.lookup_binding_partition(ex.world, GlobalRef(scope, var)) + kind = Base.binding_kind(bpart) + if kind === Base.PARTITION_KIND_GLOBAL || kind === Base.PARTITION_KIND_UNDEF_CONST || kind == Base.PARTITION_KIND_DECLARED + print(io, "\nSuggestion: add an appropriate import or assignment. This global was declared but not assigned.") + elseif kind === Base.PARTITION_KIND_FAILED + print(io, "\nHint: It looks like two or more modules export different ", + "bindings with this name, resulting in ambiguity. Try explicitly ", + "importing it from a particular module, or qualifying the name ", + "with the module it should come from.") + elseif kind === Base.PARTITION_KIND_GUARD + print(io, "\nSuggestion: check for spelling errors or missing imports.") + elseif Base.is_some_imported(kind) + print(io, "\nSuggestion: this global was defined as `$(Base.partition_restriction(bpart).globalref)` but not assigned a value.") + end + elseif scope === :static_parameter + print(io, "\nSuggestion: run Test.detect_unbound_args to detect method arguments that do not fully constrain a type parameter.") + elseif scope === :local + print(io, "\nSuggestion: check for an assignment to a local variable that shadows a global of the same name.") + end + else + scope = undef + end + if scope !== Base + warned = _UndefVarError_warnfor(io, [Base], var) + + if !warned + modules_to_check = (m for m in Base.loaded_modules_order + if m !== Core && m !== Base && m !== Main && m !== scope) + warned |= _UndefVarError_warnfor(io, modules_to_check, var) + end + + warned || _UndefVarError_warnfor(io, [Core, Main], var) + end + return nothing +end + +function _UndefVarError_warnfor(io::IO, modules, var::Symbol) + active_mod = Base.active_module() + + warned = false + # collect modules which export or make public the variable by + # the module in which the variable is defined + to_warn_about = Dict{Module, Vector{Module}}() + for m in modules + # only include in info if binding has a value and is exported or public + if !Base.isdefined(m, var) || (!Base.isexported(m, var) && !Base.ispublic(m, var)) + continue + end + warned = true + + # handle case where the undefined variable is the name of a loaded module + if Symbol(m) == var && !isdefined(active_mod, var) + print(io, "\nHint: $m is loaded but not imported in the active module $active_mod.") + continue + end + + binding_m = Base.binding_module(m, var) + if !haskey(to_warn_about, binding_m) + to_warn_about[binding_m] = [m] + else + push!(to_warn_about[binding_m], m) + end + end + + for (binding_m, modules) in pairs(to_warn_about) + print(io, "\nHint: a global variable of this name also exists in ", binding_m, ".") + for m in modules + m == binding_m && continue + how_available = if Base.isexported(m, var) + "exported by" + elseif Base.ispublic(m, var) + "declared public in" + end + print(io, "\n - Also $how_available $m") + if !isdefined(active_mod, nameof(m)) || (getproperty(active_mod, nameof(m)) !== m) + print(io, " (loaded but not imported in $active_mod)") + end + print(io, ".") + end + end + return warned +end + +Base.Experimental.register_error_hint(UndefVarError_hint, UndefVarError) + # ExceptionStack implementation size(s::ExceptionStack) = size(s.stack) getindex(s::ExceptionStack, i::Int) = s.stack[i] diff --git a/stdlib/REPL/src/REPL.jl b/stdlib/REPL/src/REPL.jl index 99142d8a64558..c621fbbb0836e 100644 --- a/stdlib/REPL/src/REPL.jl +++ b/stdlib/REPL/src/REPL.jl @@ -17,7 +17,7 @@ module REPL Base.Experimental.@optlevel 1 Base.Experimental.@max_methods 1 -function UndefVarError_hint(io::IO, ex::UndefVarError) +function UndefVarError_REPL_hint(io::IO, ex::UndefVarError) var = ex.var if var === :or print(io, "\nSuggestion: Use `||` for short-circuiting boolean OR.") @@ -30,95 +30,11 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) elseif var === :quit print(io, "\nSuggestion: To exit Julia, use Ctrl-D, or type exit() and press enter.") end - if isdefined(ex, :scope) - scope = ex.scope - if scope isa Module - bpart = Base.lookup_binding_partition(ex.world, GlobalRef(scope, var)) - kind = Base.binding_kind(bpart) - if kind === Base.PARTITION_KIND_GLOBAL || kind === Base.PARTITION_KIND_UNDEF_CONST || kind == Base.PARTITION_KIND_DECLARED - print(io, "\nSuggestion: add an appropriate import or assignment. This global was declared but not assigned.") - elseif kind === Base.PARTITION_KIND_FAILED - print(io, "\nHint: It looks like two or more modules export different ", - "bindings with this name, resulting in ambiguity. Try explicitly ", - "importing it from a particular module, or qualifying the name ", - "with the module it should come from.") - elseif kind === Base.PARTITION_KIND_GUARD - print(io, "\nSuggestion: check for spelling errors or missing imports.") - elseif Base.is_some_imported(kind) - print(io, "\nSuggestion: this global was defined as `$(Base.partition_restriction(bpart).globalref)` but not assigned a value.") - end - elseif scope === :static_parameter - print(io, "\nSuggestion: run Test.detect_unbound_args to detect method arguments that do not fully constrain a type parameter.") - elseif scope === :local - print(io, "\nSuggestion: check for an assignment to a local variable that shadows a global of the same name.") - end - else - scope = undef - end - if scope !== Base - warned = _UndefVarError_warnfor(io, [Base], var) - - if !warned - modules_to_check = (m for m in Base.loaded_modules_order - if m !== Core && m !== Base && m !== Main && m !== scope) - warned |= _UndefVarError_warnfor(io, modules_to_check, var) - end - - warned || _UndefVarError_warnfor(io, [Core, Main], var) - end - return nothing -end - -function _UndefVarError_warnfor(io::IO, modules, var::Symbol) - active_mod = Base.active_module() - - warned = false - # collect modules which export or make public the variable by - # the module in which the variable is defined - to_warn_about = Dict{Module, Vector{Module}}() - for m in modules - # only include in info if binding has a value and is exported or public - if !Base.isdefined(m, var) || (!Base.isexported(m, var) && !Base.ispublic(m, var)) - continue - end - warned = true - - # handle case where the undefined variable is the name of a loaded module - if Symbol(m) == var && !isdefined(active_mod, var) - print(io, "\nHint: $m is loaded but not imported in the active module $active_mod.") - continue - end - - binding_m = Base.binding_module(m, var) - if !haskey(to_warn_about, binding_m) - to_warn_about[binding_m] = [m] - else - push!(to_warn_about[binding_m], m) - end - end - - for (binding_m, modules) in pairs(to_warn_about) - print(io, "\nHint: a global variable of this name also exists in ", binding_m, ".") - for m in modules - m == binding_m && continue - how_available = if Base.isexported(m, var) - "exported by" - elseif Base.ispublic(m, var) - "declared public in" - end - print(io, "\n - Also $how_available $m") - if !isdefined(active_mod, nameof(m)) || (getproperty(active_mod, nameof(m)) !== m) - print(io, " (loaded but not imported in $active_mod)") - end - print(io, ".") - end - end - return warned end function __init__() Base.REPL_MODULE_REF[] = REPL - Base.Experimental.register_error_hint(UndefVarError_hint, UndefVarError) + Base.Experimental.register_error_hint(UndefVarError_REPL_hint, UndefVarError) return nothing end diff --git a/stdlib/REPL/test/repl.jl b/stdlib/REPL/test/repl.jl index a0bfa9220f97b..241464ca48942 100644 --- a/stdlib/REPL/test/repl.jl +++ b/stdlib/REPL/test/repl.jl @@ -1832,85 +1832,6 @@ fake_repl() do stdin_write, stdout_read, repl @test contains(txt, "Some type information was truncated. Use `show(err)` to see complete types.") end -try # test the functionality of `UndefVarError_hint` - @assert isempty(Base.Experimental._hint_handlers) - Base.Experimental.register_error_hint(REPL.UndefVarError_hint, UndefVarError) - - fake_repl() do stdin_write, stdout_read, repl - backend = REPL.REPLBackend() - repltask = @async REPL.run_repl(repl; backend) - write(stdin_write, """ - module A53000 - export f - f() = 0.0 - end - - module C_outer_53000 - import ..A53000: f - public f - - module C_inner_53000 - import ..C_outer_53000: f - export f - end - end - - module D_53000 - public f - f() = 1.0 - end - - C_inner_53000 = "I'm a decoy with the same name as C_inner_53000!" - - append!(Base.loaded_modules_order, [A53000, C_outer_53000, C_outer_53000.C_inner_53000, D_53000]) - f - """ - ) - write(stdin_write, "\nZZZZZ\n") - txt = readuntil(stdout_read, "ZZZZZ") - write(stdin_write, '\x04') - wait(repltask) - @test occursin("Hint: a global variable of this name also exists in Main.A53000.", txt) - @test occursin("Hint: a global variable of this name also exists in Main.D_53000.", txt) - @test occursin("- Also declared public in Main.C_outer_53000.", txt) - @test occursin("- Also exported by Main.C_outer_53000.C_inner_53000 (loaded but not imported in Main).", txt) - end -catch e - # fail test if error - @test false -finally - empty!(Base.Experimental._hint_handlers) -end - -try # test the functionality of `UndefVarError_hint` against import clashes - @assert isempty(Base.Experimental._hint_handlers) - Base.Experimental.register_error_hint(REPL.UndefVarError_hint, UndefVarError) - - @eval module X - - module A - export x - x = 1 - end # A - - module B - export x - x = 2 - end # B - - using .A, .B - - end # X - - expected_message = string("\nHint: It looks like two or more modules export different ", - "bindings with this name, resulting in ambiguity. Try explicitly ", - "importing it from a particular module, or qualifying the name ", - "with the module it should come from.") - @test_throws expected_message X.x -finally - empty!(Base.Experimental._hint_handlers) -end - # Hints for tab completes fake_repl() do stdin_write, stdout_read, repl diff --git a/test/errorshow.jl b/test/errorshow.jl index 8e13d0242ae35..ef65d70513c0d 100644 --- a/test/errorshow.jl +++ b/test/errorshow.jl @@ -12,7 +12,6 @@ Base.Experimental.register_error_hint(Base.methods_on_iterable, MethodError) Base.Experimental.register_error_hint(Base.nonsetable_type_hint_handler, MethodError) Base.Experimental.register_error_hint(Base.fielderror_listfields_hint_handler, FieldError) Base.Experimental.register_error_hint(Base.fielderror_dict_hint_handler, FieldError) - @testset "SystemError" begin err = try; systemerror("reason", Cint(0)); false; catch ex; ex; end::SystemError errs = sprint(Base.showerror, err) @@ -878,6 +877,67 @@ end @test occursin(hintExpected, errorMsg) end +# UndefVar error hints +module A53000 + export f + f() = 0.0 +end + +module C_outer_53000 + import ..A53000: f + public f + + module C_inner_53000 + import ..C_outer_53000: f + export f + end +end + +module D_53000 + public f + f() = 1.0 +end + +C_inner_53000 = "I'm a decoy with the same name as C_inner_53000!" + +Base.Experimental.register_error_hint(Base.UndefVarError_hint, UndefVarError) + +@testset "undefvar error hints" begin + old_modules_order = Base.loaded_modules_order + append!(Base.loaded_modules_order, [A53000, C_outer_53000, C_outer_53000.C_inner_53000, D_53000]) + test = @test_throws UndefVarError f + ex = test.value::UndefVarError + errormsg = sprint(Base.showerror, ex) + mod = @__MODULE__ + @test occursin("Hint: a global variable of this name also exists in $mod.A53000.", errormsg) + @test occursin("Hint: a global variable of this name also exists in $mod.D_53000.", errormsg) + @test occursin("- Also declared public in $mod.C_outer_53000", errormsg) + @test occursin("- Also exported by $mod.C_outer_53000.C_inner_53000 (loaded but not imported in Main).", errormsg) + copy!(Base.loaded_modules_order, old_modules_order) +end +@testset " test the functionality of `UndefVarError_hint` against import clashes" begin + @eval module X + module A + export x + x = 1 + end # A + + module B + export x + x = 2 + end # B + + using .A, .B + + end # X + + expected_message = string("\nHint: It looks like two or more modules export different ", + "bindings with this name, resulting in ambiguity. Try explicitly ", + "importing it from a particular module, or qualifying the name ", + "with the module it should come from.") + @test_throws expected_message X.x +end + # test showing MethodError with type argument struct NoMethodsDefinedHere; end let buf = IOBuffer() From 5f85ec3bf5264395034cda28d30bb70664dacaab Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 13 Mar 2025 09:53:25 -0400 Subject: [PATCH 45/74] inference: exclude uncached frames from callstack (#57743) This is more consistent with how it is handled recursively and removes confusion here where the LimitedAccuracy could appear to happen from the top-level uncached frame. I was unable to make a reduced test-case showing this failing. Fixes #57634 (cherry picked from commit 28d3bd56d36ea6032027794ccc17dd5c7d0730ab) --- Compiler/src/inferencestate.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Compiler/src/inferencestate.jl b/Compiler/src/inferencestate.jl index dbad9f76852e4..6fe52ea3e6f84 100644 --- a/Compiler/src/inferencestate.jl +++ b/Compiler/src/inferencestate.jl @@ -989,12 +989,12 @@ ascending the tree from the given `AbsIntState`). Note that cycles may be visited in any order. """ struct AbsIntStackUnwind - sv::AbsIntState + callstack::Vector{AbsIntState} + AbsIntStackUnwind(sv::AbsIntState) = new(sv.callstack::Vector{AbsIntState}) end -iterate(unw::AbsIntStackUnwind) = (unw.sv, length(unw.sv.callstack::Vector{AbsIntState})) -function iterate(unw::AbsIntStackUnwind, frame::Int) +function iterate(unw::AbsIntStackUnwind, frame::Int=length(unw.callstack)) frame == 0 && return nothing - return ((unw.sv.callstack::Vector{AbsIntState})[frame], frame - 1) + return (unw.callstack[frame], frame - 1) end struct AbsIntCycle From c4b667d689572fdd823b0944053b076fe6a4d574 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 14 Mar 2025 10:25:06 -0400 Subject: [PATCH 46/74] remove old cruft from jl_set_typeinf_func (#57760) This has not actually been relevant since solving #265, so all it has been doing since then is making the build slightly slower, bigger, and less correct. (cherry picked from commit 2a168ee5a7cfa555bc7e52d180aef7bafc74b912) --- src/gf.c | 55 ------------------------------------------------------- 1 file changed, 55 deletions(-) diff --git a/src/gf.c b/src/gf.c index c07b90a5c12a2..813f9dfaca328 100644 --- a/src/gf.c +++ b/src/gf.c @@ -759,33 +759,6 @@ JL_DLLEXPORT int jl_mi_try_insert(jl_method_instance_t *mi JL_ROOTING_ARGUMENT, return ret; } -static int get_method_unspec_list(jl_typemap_entry_t *def, void *closure) -{ - size_t world = jl_atomic_load_acquire(&jl_world_counter); - jl_value_t *specializations = jl_atomic_load_relaxed(&def->func.method->specializations); - if (specializations == (jl_value_t*)jl_emptysvec) - return 1; - if (!jl_is_svec(specializations)) { - jl_method_instance_t *mi = (jl_method_instance_t*)specializations; - assert(jl_is_method_instance(mi)); - if (jl_rettype_inferred_native(mi, world, world) == jl_nothing) - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); - return 1; - } - size_t i, l = jl_svec_len(specializations); - JL_GC_PUSH1(&specializations); - for (i = 0; i < l; i++) { - jl_method_instance_t *mi = (jl_method_instance_t*)jl_svecref(specializations, i); - if ((jl_value_t*)mi != jl_nothing) { - assert(jl_is_method_instance(mi)); - if (jl_rettype_inferred_native(mi, world, world) == jl_nothing) - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); - } - } - JL_GC_POP(); - return 1; -} - int foreach_mtable_in_module( jl_module_t *m, int (*visit)(jl_methtable_t *mt, void *env), @@ -865,42 +838,14 @@ int jl_foreach_reachable_mtable(int (*visit)(jl_methtable_t *mt, void *env), voi return 1; } -static int reset_mt_caches(jl_methtable_t *mt, void *env) -{ - // removes all method caches - // this might not be entirely safe (GC or MT), thus we only do it very early in bootstrapping - if (!mt->frozen) { // make sure not to reset builtin functions - jl_atomic_store_release(&mt->leafcache, (jl_genericmemory_t*)jl_an_empty_memory_any); - jl_atomic_store_release(&mt->cache, jl_nothing); - } - jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), get_method_unspec_list, env); - return 1; -} - jl_function_t *jl_typeinf_func JL_GLOBALLY_ROOTED = NULL; JL_DLLEXPORT size_t jl_typeinf_world = 1; JL_DLLEXPORT void jl_set_typeinf_func(jl_value_t *f) { - size_t newfunc = jl_typeinf_world == 1 && jl_typeinf_func == NULL; jl_typeinf_func = (jl_function_t*)f; jl_typeinf_world = jl_get_tls_world_age(); - int world = jl_atomic_fetch_add(&jl_world_counter, 1) + 1; // make type-inference the only thing in this world - if (newfunc) { - // give type inference a chance to see all of these - // TODO: also reinfer if max_world != ~(size_t)0 - jl_array_t *unspec = jl_alloc_vec_any(0); - JL_GC_PUSH1(&unspec); - jl_foreach_reachable_mtable(reset_mt_caches, (void*)unspec); - size_t i, l; - for (i = 0, l = jl_array_nrows(unspec); i < l; i++) { - jl_method_instance_t *mi = (jl_method_instance_t*)jl_array_ptr_ref(unspec, i); - if (jl_rettype_inferred_native(mi, world, world) == jl_nothing) - jl_type_infer(mi, world, SOURCE_MODE_NOT_REQUIRED); - } - JL_GC_POP(); - } } static int very_general_type(jl_value_t *t) From 31d998e7362392ee5ed9ae2819781c8aff837bfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Sat, 15 Mar 2025 00:56:03 +0000 Subject: [PATCH 47/74] [docs] Clarify that Float16 is supported natively when possible (#57725) Co-authored-by: Jishnu Bhattacharya (cherry picked from commit 3e57a8ac3adbe28e2a9a67e2fed2a1be24375ca5) --- doc/src/manual/integers-and-floating-point-numbers.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/manual/integers-and-floating-point-numbers.md b/doc/src/manual/integers-and-floating-point-numbers.md index 0ee7850c92087..fa0ee228e873b 100644 --- a/doc/src/manual/integers-and-floating-point-numbers.md +++ b/doc/src/manual/integers-and-floating-point-numbers.md @@ -334,8 +334,8 @@ julia> typeof(x) Float64 ``` -Half-precision floating-point numbers are also supported ([`Float16`](@ref)), but they are -implemented in software and use [`Float32`](@ref) for calculations. +Half-precision floating-point numbers are also supported ([`Float16`](@ref)) on all platforms, with native instructions used on hardware which supports this number format. Otherwise, operations are implemented in software, and use [`Float32`](@ref) for intermediate calculations. +As an internal implementation detail, this is achieved under the hood by using LLVM's [`half`](https://llvm.org/docs/LangRef.html#half-precision-floating-point-intrinsics) type, which behaves similarly to what the GCC [`-fexcess-precision=16`](https://gcc.gnu.org/onlinedocs/gcc/Optimize-Options.html#index-fexcess-precision) flag does for C/C++ code. ```jldoctest julia> sizeof(Float16(4.)) From 1de014a7be7a7764b108605193894927de4dd9f7 Mon Sep 17 00:00:00 2001 From: Em Chu <61633163+mlechu@users.noreply.github.com> Date: Sat, 15 Mar 2025 00:57:22 -0700 Subject: [PATCH 48/74] lowering: Don't closure-convert in `import` or `using` (#57774) Fixes #57702. We're calling cl-convert- on `using` and `import` statements when we shouldn't, so if there's a nearby local that gets boxed (recursive function definition in this case), and the local shares a name with something in an import statement, we get a box access where we want a raw symbol. Before: ``` julia> let; let; import SHA: R; end; let; R(x...) = R(x); end; end ERROR: TypeError: in import, expected Symbol, got a value of type Expr Stacktrace: [1] top-level scope @ REPL[1]:1 ``` After: ``` julia> let; let; import SHA: R; end; let; R(x...) = R(x); end; end (::var"#R#R##0") (generic function with 1 method) ``` Previously, symbols in `import`/`using` statements would be wrapped with `outerref`, which cl-convert- wouldn't peek into. This protected us from this problem in 1.11. (cherry picked from commit ca17927311bfab2a18c9ce2a21913eb51537c2d7) --- src/julia-syntax.scm | 2 +- test/syntax.jl | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index 199119d4b1ee8..a68df3f697a8c 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -4001,7 +4001,7 @@ f(x) = yt(x) ((atom? e) e) (else (case (car e) - ((quote top core globalref thismodule lineinfo line break inert module toplevel null true false meta) e) + ((quote top core globalref thismodule lineinfo line break inert module toplevel null true false meta import using) e) ((toplevel-only) ;; hack to avoid generating a (method x) expr for struct types (if (eq? (cadr e) 'struct) diff --git a/test/syntax.jl b/test/syntax.jl index 373a9ad9e0528..dc269da0475de 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -2689,6 +2689,18 @@ import .TestImportAs.Mod2 as M2 @test !@isdefined(Mod2) @test M2 === TestImportAs.Mod2 +# 57702: nearby bindings shouldn't cause us to closure-convert in import/using +module OddImports +using Test +module ABC end +x = let; let; import .ABC; end; let; ABC() = (ABC,); end; end +y = let; let; using .ABC; end; let; ABC() = (ABC,); end; end +z = let; let; import SHA: R; end; let; R(x...) = R(x); end; end +@test x isa Function +@test y isa Function +@test z isa Function +end + @testset "unicode modifiers after '" begin @test Meta.parse("a'ᵀ") == Expr(:call, Symbol("'ᵀ"), :a) @test Meta.parse("a'⁻¹") == Expr(:call, Symbol("'⁻¹"), :a) From 4a420a181e84be2e6c3e6f07a1648951478d55ea Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Sat, 15 Mar 2025 04:12:39 -0400 Subject: [PATCH 49/74] fix missing `!` in a strip-metadata case (#57778) (cherry picked from commit e14fb0ced3702e4ce54031b78dc90ce0557efe10) --- src/staticdata.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/staticdata.c b/src/staticdata.c index b450cf2561107..ec6d706bace9f 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -795,7 +795,7 @@ static void jl_queue_module_for_serialization(jl_serializer_state *s, jl_module_ { jl_queue_for_serialization(s, m->name); jl_queue_for_serialization(s, m->parent); - if (jl_options.strip_metadata) + if (!jl_options.strip_metadata) jl_queue_for_serialization(s, m->file); if (jl_options.trim) { jl_queue_for_serialization_(s, (jl_value_t*)jl_atomic_load_relaxed(&m->bindings), 0, 1); From dd14c58ad0cb1f1dd1677d24a377289846c0c694 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sat, 15 Mar 2025 07:53:13 -0400 Subject: [PATCH 50/74] print admonition for auto-import only once per module (#57775) Fix negation mistake introduced by #57614 which broke #57378. Re-fix #57756 (cherry picked from commit 744e03dcecacbb3c00efa10e46ca4bbdb5fbfef4) --- src/module.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/module.c b/src/module.c index 572f5fd7e8142..291ab26b98509 100644 --- a/src/module.c +++ b/src/module.c @@ -680,7 +680,7 @@ static NOINLINE void print_backdate_admonition(jl_binding_t *b) JL_NOTSAFEPOINT static inline void check_backdated_binding(jl_binding_t *b, enum jl_partition_kind kind) JL_NOTSAFEPOINT { if (__unlikely(kind == PARTITION_KIND_BACKDATED_CONST) && - !(jl_atomic_fetch_or(&b->flags, BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION) & BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION)) { + !(jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION) & BINDING_FLAG_DID_PRINT_BACKDATE_ADMONITION)) { print_backdate_admonition(b); } } @@ -858,8 +858,8 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ jl_errorf("invalid method definition in %s: function %s.%s must be explicitly imported to be extended", jl_module_debug_name(m), jl_module_debug_name(from), jl_symbol_name(var)); } - else if (jl_atomic_fetch_or(&b->flags, BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION) & - BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION) { + else if (!(jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION) & + BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION)) { jl_printf(JL_STDERR, "WARNING: Constructor for type \"%s\" was extended in `%s` without explicit qualification or import.\n" " NOTE: Assumed \"%s\" refers to `%s.%s`. This behavior is deprecated and may differ in future versions.`\n" " NOTE: This behavior may have differed in Julia versions prior to 1.12.\n" From 02b8670a7e43ec5c9430c055977226e9a8e17608 Mon Sep 17 00:00:00 2001 From: Jeff Bezanson Date: Sat, 15 Mar 2025 09:33:09 -0400 Subject: [PATCH 51/74] enable JuliaSyntax parser for user code in juliac (#57777) This was an oversight. (cherry picked from commit 17f99bc1c7733658ab784d5c00eb28360c7578fa) --- contrib/juliac-buildscript.jl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index 459922ab1ccb1..6be2d20597079 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -17,6 +17,9 @@ task.rngState3 = 0x3a77f7189200c20b task.rngState4 = 0x5502376d099035ae uuid_tuple = (UInt64(0), UInt64(0)) ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple) +if Base.get_bool_env("JULIA_USE_FLISP_PARSER", false) === false + Base.JuliaSyntax.enable_in_core!() +end # Patch methods in Core and Base From 84bc0c6ef58bf0eee6191ca77b4474c97f4bb08e Mon Sep 17 00:00:00 2001 From: Jakob Nybo Nissen Date: Mon, 17 Mar 2025 03:54:52 +0100 Subject: [PATCH 52/74] Bugfix: String indexing in big_str macro (#57621) Make sure that non-ASCII in the big string macro correctly throws an ArgumentError instead of a string indexing error. (cherry picked from commit 246c93b5ac3aad1f6d25fdce4c57e964ff6c91e5) --- base/int.jl | 7 ++++--- test/int.jl | 4 ++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/base/int.jl b/base/int.jl index 8a80f90f7e2c1..a0ab68436145d 100644 --- a/base/int.jl +++ b/base/int.jl @@ -699,13 +699,14 @@ macro big_str(s) message = "invalid number format $s for BigInt or BigFloat" throw_error = :(throw(ArgumentError($message))) if '_' in s - # remove _ in s[2:end-1] - bf = IOBuffer(maxsize=lastindex(s)) + # remove _ in s[2:end-1]. + # Do not allow '_' right before or after dot. + bf = IOBuffer(sizehint=ncodeunits(s)) c = s[1] print(bf, c) is_prev_underscore = (c == '_') is_prev_dot = (c == '.') - for c in SubString(s, 2, lastindex(s)-1) + for c in SubString(s, nextind(s, 1), prevind(s, lastindex(s))) c != '_' && print(bf, c) c == '_' && is_prev_dot && return throw_error c == '.' && is_prev_underscore && return throw_error diff --git a/test/int.jl b/test/int.jl index f79bc5a9781d0..dda736cd7a9d0 100644 --- a/test/int.jl +++ b/test/int.jl @@ -118,6 +118,10 @@ end @test big"1.0" == BigFloat(1.0) @test_throws ArgumentError big"1.0.3" @test_throws ArgumentError big"pi" + + @test_throws ArgumentError big"_æ1" + @test_throws ArgumentError big"æ_1" + @test_throws ArgumentError big"_ææ" end @test round(UInt8, 123) == 123 From 2916acdfd148c56f70315c770fe445152417d84f Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Mon, 17 Mar 2025 06:05:16 +0100 Subject: [PATCH 53/74] fix special function `::Real` fallback stack overflow (#57790) Fixes #57789 (cherry picked from commit 6817691ecbba3e2a687348c085af1c3d76f020fe) --- base/math.jl | 2 +- test/math.jl | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/base/math.jl b/base/math.jl index 0e5f9dd41bdca..beef8de09bd8b 100644 --- a/base/math.jl +++ b/base/math.jl @@ -1542,7 +1542,7 @@ for f in (:sin, :cos, :tan, :asin, :atan, :acos, :exponent, :sqrt, :cbrt, :sinpi, :cospi, :sincospi, :tanpi) @eval function ($f)(x::Real) xf = float(x) - x === xf && throw(MethodError($f, (x,))) + xf isa typeof(x) && throw(MethodError($f, (x,))) return ($f)(xf) end @eval $(f)(::Missing) = missing diff --git a/test/math.jl b/test/math.jl index 8f5ada013557c..75ee928c62e65 100644 --- a/test/math.jl +++ b/test/math.jl @@ -1549,6 +1549,14 @@ end end end +@testset "special function `::Real` fallback shouldn't recur without bound, issue #57789" begin + mutable struct Issue57789 <: Real end + Base.float(::Issue57789) = Issue57789() + for f ∈ (sin, sinpi, log, exp) + @test_throws MethodError f(Issue57789()) + end +end + # Test that sqrt behaves correctly and doesn't exhibit fp80 double rounding. # This happened on old glibc versions. # Test case from https://sourceware.org/bugzilla/show_bug.cgi?id=14032. From 09586ff7509fd47a87f20b5a967d5f4e31277e66 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Mon, 17 Mar 2025 16:53:43 -0400 Subject: [PATCH 54/74] stackwalk: fix heuristic termination (#57801) When getting stacktraces on non-X86 platforms, the first frame may not have been set up yet, incorrectly triggering this bad-frame detection logic. This should fix the issue of async unwind failing after only getting 2 frames, if the first frame happens to land in the function header. This is not normally an issue on X86 or non-signals, but also causes no expected issues to be the same logic there too. Fix #52334 After (on arm64-apple-darwin24.3.0): ``` julia> f(1) Warning: detected a stack overflow; program state may be corrupted, so further execution might be unreliable. ERROR: StackOverflowError: Stacktrace: [1] f(x::Int64) @ Main ./REPL[3]:1 [2] g(x::Int64) @ Main ./REPL[4]:1 --- the above 2 lines are repeated 39990 more times --- [79983] f(x::Int64) @ Main ./REPL[3]:1 ``` n.b. This will not fix and is not related to any issues where profiling gets only a single stack frame during profiling of syscalls on Apple AArch64. This fix is specific to the bug where it gets exactly 2 frames. (cherry picked from commit f82917a8ada5a3f4e4ca85408e078a6389c9e94a) --- src/stackwalk.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/stackwalk.c b/src/stackwalk.c index 14dc5709671dc..1f3c8d690c8ce 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -98,9 +98,13 @@ static int jl_unw_stepn(bt_cursor_t *cursor, jl_bt_element_t *bt_data, size_t *b } uintptr_t oldsp = thesp; have_more_frames = jl_unw_step(cursor, from_signal_handler, &return_ip, &thesp); - if (oldsp >= thesp && !jl_running_under_rr(0)) { - // The stack pointer is clearly bad, as it must grow downwards. + if ((n < 2 ? oldsp > thesp : oldsp >= thesp) && !jl_running_under_rr(0)) { + // The stack pointer is clearly bad, as it must grow downwards, // But sometimes the external unwinder doesn't check that. + // Except for n==0 when there is no oldsp and n==1 on all platforms but i686/x86_64. + // (on x86, the platform first pushes the new stack frame, then does the + // call, on almost all other platforms, the platform first does the call, + // then the user pushes the link register to the frame). have_more_frames = 0; } if (return_ip == 0) { @@ -132,11 +136,11 @@ static int jl_unw_stepn(bt_cursor_t *cursor, jl_bt_element_t *bt_data, size_t *b // * The way that libunwind handles it in `unw_get_proc_name`: // https://lists.nongnu.org/archive/html/libunwind-devel/2014-06/msg00025.html uintptr_t call_ip = return_ip; + #if defined(_CPU_ARM_) // ARM instruction pointer encoding uses the low bit as a flag for // thumb mode, which must be cleared before further use. (Note not // needed for ARM AArch64.) See // https://github.com/libunwind/libunwind/pull/131 - #ifdef _CPU_ARM_ call_ip &= ~(uintptr_t)0x1; #endif // Now there's two main cases to adjust for: From 65b5aa799cca80f4dbfcad4b037fbcfef346dffe Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Mon, 17 Mar 2025 20:42:16 -0700 Subject: [PATCH 55/74] Fix fptrunc Float64 -> Float16 rounding through Float32 (#57809) Widening from Float32 to Float64 and then rounding to Float16 will not introduce any error, but going from Float64 -> Float32 -> Float16 will round incorrectly if the intermediate Float32 is halfway between two Float16s. Fixes #57805. Thanks to @vtjnash for suggesting the fix. Co-authored-by: Jameson Nash (cherry picked from commit a676b12a829216519fe651beadf679421ceb500f) --- src/runtime_intrinsics.c | 4 ++-- test/intrinsics.jl | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index f5b281f9e92ed..09f55f1d1f1d8 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -1629,8 +1629,8 @@ cvt_iintrinsic(LLVMFPtoUI, fptoui) #define fintrinsic_read_float32(p) *(float *)p #define fintrinsic_read_float64(p) *(double *)p -#define fintrinsic_write_float16(p, x) *(uint16_t *)p = float_to_half(x) -#define fintrinsic_write_bfloat16(p, x) *(uint16_t *)p = float_to_bfloat(x) +#define fintrinsic_write_float16(p, x) *(uint16_t *)p = double_to_half(x) +#define fintrinsic_write_bfloat16(p, x) *(uint16_t *)p = double_to_bfloat(x) #define fintrinsic_write_float32(p, x) *(float *)p = x #define fintrinsic_write_float64(p, x) *(double *)p = x diff --git a/test/intrinsics.jl b/test/intrinsics.jl index 12867908bf5a4..ee1da6d7a45ae 100644 --- a/test/intrinsics.jl +++ b/test/intrinsics.jl @@ -304,6 +304,17 @@ end @test_intrinsic Core.Intrinsics.fptrunc Float16 Float32(3.3) Float16(3.3) @test_intrinsic Core.Intrinsics.fptrunc Float16 Float64(3.3) Float16(3.3) + # #57805 - cases where rounding Float64 -> Float32 -> Float16 would fail + # 2^-25 * 0b1.0000000000000000000000000000000000000001 binary + # 0 01111100110 0000000000000000000000000000000000000001000000000000 + # 2^-25 * 0b1.0 binary + # 0 01100110 00000000000000000000000 + # 2^-14 * 0b0.0000000001 (subnormal) + # 0 00000 0000000001 (correct) + # 0 00000 0000000000 (incorrect) + @test_intrinsic Core.Intrinsics.fptrunc Float16 0x1.0000000001p-25 Float16(6.0e-8) + @test_intrinsic Core.Intrinsics.fptrunc Float16 -0x1.0000000001p-25 Float16(-6.0e-8) + # float_to_half/bfloat_to_float special cases @test_intrinsic Core.Intrinsics.fptrunc Float16 Inf32 Inf16 @test_intrinsic Core.Intrinsics.fptrunc Float16 -Inf32 -Inf16 From 3778b8003b838aaaf2efe2db84eb5e9723af690f Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 19 Mar 2025 09:41:42 -0400 Subject: [PATCH 56/74] add missing wb in jl_maybe_reresolve_implicit (#57804) Also stop using a custom rooting strategy that is not understandable by the static analysis. Unfortunately, we currently have no checker for missing jl_gc_wb calls however. Fix #57700 (cherry picked from commit abb1313f51b50c136d48084c6a9a4ca341953df4) --- src/julia_internal.h | 3 +-- src/module.c | 28 ++++++++++++++++------------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/julia_internal.h b/src/julia_internal.h index a5b7be7bba0b6..f358d30b9c193 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -855,8 +855,7 @@ typedef struct _modstack_t { jl_binding_t *b; struct _modstack_t *prev; } modstack_t; -void jl_check_new_binding_implicit( - jl_binding_partition_t *new_bpart JL_MAYBE_UNROOTED, jl_binding_t *b, modstack_t *st, size_t world); +void jl_check_new_binding_implicit(jl_binding_partition_t *new_bpart, jl_binding_t *b, modstack_t *st, size_t world); #ifndef __clang_gcanalyzer__ // The analyzer doesn't like looking through the arraylist, so just model the diff --git a/src/module.c b/src/module.c index 291ab26b98509..1e138980a0dc0 100644 --- a/src/module.c +++ b/src/module.c @@ -47,7 +47,7 @@ static int eq_bindings(jl_binding_partition_t *owner, jl_binding_t *alias, size_ // find a binding from a module's `usings` list void jl_check_new_binding_implicit( - jl_binding_partition_t *new_bpart JL_MAYBE_UNROOTED, jl_binding_t *b, modstack_t *st, size_t world) + jl_binding_partition_t *new_bpart, jl_binding_t *b, modstack_t *st, size_t world) { modstack_t top = { b, st }; modstack_t *tmp = st; @@ -59,7 +59,6 @@ void jl_check_new_binding_implicit( } } - JL_GC_PUSH1(&new_bpart); jl_module_t *m = b->globalref->mod; jl_sym_t *var = b->globalref->name; @@ -164,8 +163,6 @@ void jl_check_new_binding_implicit( new_bpart->kind = guard_kind; new_bpart->restriction = NULL; } - JL_GC_POP(); - return; } JL_DLLEXPORT jl_binding_partition_t *jl_maybe_reresolve_implicit(jl_binding_t *b, size_t new_max_world) @@ -173,22 +170,26 @@ JL_DLLEXPORT jl_binding_partition_t *jl_maybe_reresolve_implicit(jl_binding_t *b jl_binding_partition_t *new_bpart = new_binding_partition(); jl_binding_partition_t *bpart = jl_atomic_load_acquire(&b->partitions); assert(bpart); + JL_GC_PUSH1(&new_bpart); while (1) { jl_atomic_store_relaxed(&new_bpart->next, bpart); jl_gc_wb(new_bpart, bpart); jl_check_new_binding_implicit(new_bpart, b, NULL, new_max_world+1); - JL_GC_PROMISE_ROOTED(new_bpart); // TODO: Analyzer doesn't understand MAYBE_UNROOTED properly if (bpart->kind & PARTITION_FLAG_EXPORTED) new_bpart->kind |= PARTITION_FLAG_EXPORTED; - if (new_bpart->kind == bpart->kind && new_bpart->restriction == bpart->restriction) + if (new_bpart->kind == bpart->kind && new_bpart->restriction == bpart->restriction) { + JL_GC_POP(); return bpart; + } // Resolution changed, insert the new partition size_t expected_max_world = ~(size_t)0; if (jl_atomic_cmpswap(&bpart->max_world, &expected_max_world, new_max_world) && - jl_atomic_cmpswap(&b->partitions, &bpart, new_bpart)) - break; + jl_atomic_cmpswap(&b->partitions, &bpart, new_bpart)) { + jl_gc_wb(b, new_bpart); + JL_GC_POP(); + return new_bpart; + } } - return new_bpart; } STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, jl_value_t *parent, _Atomic(jl_binding_partition_t *)*insert, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED @@ -197,6 +198,7 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b jl_binding_partition_t *bpart = jl_atomic_load_relaxed(insert); size_t max_world = (size_t)-1; jl_binding_partition_t *new_bpart = NULL; + JL_GC_PUSH1(&new_bpart); while (1) { while (bpart && world < bpart->min_world) { insert = &bpart->next; @@ -204,8 +206,10 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b parent = (jl_value_t *)bpart; bpart = jl_atomic_load_relaxed(&bpart->next); } - if (bpart && world <= jl_atomic_load_relaxed(&bpart->max_world)) + if (bpart && world <= jl_atomic_load_relaxed(&bpart->max_world)) { + JL_GC_POP(); return bpart; + } if (!new_bpart) new_bpart = new_binding_partition(); jl_atomic_store_relaxed(&new_bpart->next, bpart); @@ -213,12 +217,12 @@ STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b jl_gc_wb(new_bpart, bpart); // Not fresh the second time around the loop new_bpart->min_world = bpart ? jl_atomic_load_relaxed(&bpart->max_world) + 1 : 0; jl_atomic_store_relaxed(&new_bpart->max_world, max_world); - JL_GC_PROMISE_ROOTED(new_bpart); // TODO: Analyzer doesn't understand MAYBE_UNROOTED properly jl_check_new_binding_implicit(new_bpart, b, st, world); if (bpart && (bpart->kind & PARTITION_FLAG_EXPORTED)) new_bpart->kind |= PARTITION_FLAG_EXPORTED; if (jl_atomic_cmpswap(insert, &bpart, new_bpart)) { jl_gc_wb(parent, new_bpart); + JL_GC_POP(); return new_bpart; } } @@ -1435,7 +1439,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, new_bpart->min_world = new_world; if ((kind & PARTITION_MASK_KIND) == PARTITION_KIND_IMPLICIT_RECOMPUTE) { assert(!restriction_val); - jl_check_new_binding_implicit(new_bpart /* callee rooted */, b, NULL, new_world); + jl_check_new_binding_implicit(new_bpart, b, NULL, new_world); new_bpart->kind |= kind & PARTITION_MASK_FLAG; if (new_bpart->kind == old_bpart->kind && new_bpart->restriction == old_bpart->restriction) { JL_GC_POP(); From 59ad0917394b0adcd0ce1677ee6aeb0697c8408d Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 20 Mar 2025 15:55:19 -0300 Subject: [PATCH 57/74] Fix OncePerThread issue when building a sysimage using the current sysimage (#57656) This is quite tricky to test unfortunately, but https://github.com/JuliaLang/julia/pull/57544 caught this and this fixes that --------- Co-authored-by: Jameson Nash (cherry picked from commit bf01638640659f8223b6e02c7eed9b71a7fd09c5) --- base/lock.jl | 18 ++++++++------- test/compileall.jl | 56 +++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 63 insertions(+), 11 deletions(-) diff --git a/base/lock.jl b/base/lock.jl index 06c35f989422d..40bc9e08bd9b0 100644 --- a/base/lock.jl +++ b/base/lock.jl @@ -702,10 +702,6 @@ mutable struct OncePerProcess{T, F} <: Function function OncePerProcess{T,F}(initializer::F) where {T, F} once = new{T,F}(nothing, PerStateInitial, true, initializer, ReentrantLock()) - ccall(:jl_set_precompile_field_replace, Cvoid, (Any, Any, Any), - once, :value, nothing) - ccall(:jl_set_precompile_field_replace, Cvoid, (Any, Any, Any), - once, :state, PerStateInitial) return once end end @@ -721,6 +717,10 @@ OncePerProcess(initializer) = OncePerProcess{Base.promote_op(initializer), typeo try state = @atomic :monotonic once.state if state == PerStateInitial + ccall(:jl_set_precompile_field_replace, Cvoid, (Any, Any, Any), + once, :value, nothing) + ccall(:jl_set_precompile_field_replace, Cvoid, (Any, Any, Any), + once, :state, PerStateInitial) once.value = once.initializer() elseif state == PerStateErrored error("OncePerProcess initializer failed previously") @@ -809,10 +809,6 @@ mutable struct OncePerThread{T, F} <: Function function OncePerThread{T,F}(initializer::F) where {T, F} xs, ss = AtomicMemory{T}(), AtomicMemory{UInt8}() once = new{T,F}(xs, ss, initializer) - ccall(:jl_set_precompile_field_replace, Cvoid, (Any, Any, Any), - once, :xs, xs) - ccall(:jl_set_precompile_field_replace, Cvoid, (Any, Any, Any), - once, :ss, ss) return once end end @@ -849,6 +845,12 @@ OncePerThread(initializer) = OncePerThread{Base.promote_op(initializer), typeof( ss = @atomic :monotonic once.ss xs = @atomic :monotonic once.xs if tid > length(ss) + if length(ss) == 0 # We are the first to initialize + ccall(:jl_set_precompile_field_replace, Cvoid, (Any, Any, Any), + once, :xs, xs) + ccall(:jl_set_precompile_field_replace, Cvoid, (Any, Any, Any), + once, :ss, ss) + end @assert len <= length(ss) <= length(newss) "logical constraint violation" fill_monotonic!(newss, PerStateInitial) xs = copyto_monotonic!(newxs, xs) diff --git a/test/compileall.jl b/test/compileall.jl index beec0d6df49ab..1987bda7f04df 100644 --- a/test/compileall.jl +++ b/test/compileall.jl @@ -2,10 +2,60 @@ # We make it a separate test target here, so that it can run in parallel # with the rest of the tests. -mktempdir() do dir - @test success(pipeline(`$(Base.julia_cmd()) --compile=all --strip-ir --output-o $(dir)/sys.o.a -e 'exit()'`, stderr=stderr)) skip=(Sys.WORD_SIZE == 32) +function precompile_test_harness(@nospecialize(f)) + load_path = mktempdir() + try + pushfirst!(LOAD_PATH, load_path) + pushfirst!(DEPOT_PATH, load_path) + f(load_path) + finally + try + rm(load_path, force=true, recursive=true) + catch err + @show err + end + filter!((≠)(load_path), LOAD_PATH) + filter!((≠)(load_path), DEPOT_PATH) + end + return nothing +end + +precompile_test_harness() do dir + Foo_file = joinpath(dir, "OncePerFoo.jl") + image_file = joinpath(dir, "img.jl") + write(Foo_file, + """module OncePerFoo + + const f = OncePerThread{Nothing}() do + println(Core.stdout, "Running thread init...") + end + + f() # Executed during pre-compilation + + end # module OncePerFoo + """) + + write(image_file, + """ + Base.init_depot_path() + Base.init_load_path() + using OncePerFoo + + function main() + OncePerFoo.f() + return 0 + end + + OncePerFoo.f() # fire init during compilation time + + """) + Base.compilecache(Base.PkgId("OncePerFoo")) + new_env = Dict(["JULIA_DEPOT_PATH" => join(DEPOT_PATH, Sys.iswindows() ? ';' : ':'), + "JULIA_LOAD_PATH" => join(LOAD_PATH, Sys.iswindows() ? ';' : ':')]) + @test success(pipeline(addenv(`$(Base.julia_cmd()) --compile=all -t1,0 --strip-ir --output-o $(dir)/sys.o.a $(image_file) `, new_env), stderr=stderr, stdout=stdout)) skip=(Sys.WORD_SIZE == 32) if isfile(joinpath(dir, "sys.o.a")) Base.Linking.link_image(joinpath(dir, "sys.o.a"), joinpath(dir, "sys.so")) - @test success(`$(Base.julia_cmd()) -J $(dir)/sys.so -e 'Base.scrub_repl_backtrace(nothing); exit()'`) + str = readchomp(`$(Base.julia_cmd()) -t1,0 -J $(dir)/sys.so -e 'Base.scrub_repl_backtrace(nothing); println("loaded"); main()'`) + @test split(str, '\n') == ["loaded", "Running thread init..."] end end From 2a04708a5267a59e4d6b5ddfbfefd2ad80e43a50 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 20 Mar 2025 09:33:31 -0400 Subject: [PATCH 58/74] Fix error message when using `const` inside a function, require the LHS to be global, and prohibit "const x[] = 1" (#57470) The changes to `const` resulted in confusing error messages when it was used inside a function (#57334). On 1.11.3: ``` julia> function f() const x = 1 end ERROR: syntax: unsupported `const` declaration on local variable around REPL[1]:2 Stacktrace: [1] top-level scope @ REPL[1]:1 ``` On nightly: ``` julia> function f() const x = 1 end ERROR: syntax: World age increment not at top level Stacktrace: [1] top-level scope @ REPL[1]:1 ``` In prior versions, we also accepted confused expressions like: ``` x = Ref(1) const x[] = 1 ``` This change adds a new error messages explicitly prohibiting `const` where the left hand side is not declaring variables: ``` ERROR: syntax: `const` left hand side "x[]" contains non-variables around REPL[2]:1 Stacktrace: [1] top-level scope @ REPL[2]:1 ``` Finally, #54773 made `const` stop participating in scope resolution (the left hand side was always taken to be in global scope). Some expressions that were prohibited started being accepted: In 1.11.3: ``` julia> let const x = 1 end ERROR: syntax: unsupported `const` declaration on local variable around REPL[1]:2 Stacktrace: [1] top-level scope @ REPL[1]:1 ``` Nightly: ``` julia> let const x = 1 end 1 ``` This change rejects `const` unless the variable would be in global scope (`global const` would be required in the example), so we don't lose the ability to make `const` in local scope meaningful later. (cherry picked from commit fb01f9159cb050a207bedc56d36a3f9cce5d0ffb) --- src/julia-syntax.scm | 388 ++++++++++++++++++++++++------------------- test/syntax.jl | 104 +++++++++++- 2 files changed, 315 insertions(+), 177 deletions(-) diff --git a/src/julia-syntax.scm b/src/julia-syntax.scm index a68df3f697a8c..8bc313610bbfa 100644 --- a/src/julia-syntax.scm +++ b/src/julia-syntax.scm @@ -1429,7 +1429,7 @@ (else (error "invalid \"try\" form"))))) -(define (expand-unionall-def name type-ex (allow-local #t)) +(define (expand-unionall-def name type-ex (const? #t)) (if (and (pair? name) (eq? (car name) 'curly)) (let ((name (cadr name)) @@ -1440,7 +1440,7 @@ (expand-forms `(block (= ,rr (where ,type-ex ,@params)) - (,(if allow-local 'assign-const-if-global 'const) ,name ,rr) + (,(if const? 'const 'assign-const-if-global) ,name ,rr) (latestworld-if-toplevel) ,rr))) (expand-forms @@ -1450,38 +1450,24 @@ (filter (lambda (x) (not (underscore-symbol? x))) syms)) ;; Expand `[global] const a::T = val` -(define (expand-const-decl e (mustassgn #f)) - (if (length= e 3) e - (let ((arg (cadr e))) - (if (atom? arg) - (if mustassgn - (error "expected assignment after \"const\"") - e) - (case (car arg) - ((global) - (expand-const-decl `(const ,(cadr arg)) #t)) - ((=) - (cond - ;; `const f() = ...` - The `const` here is inoperative, but the syntax happened to work in earlier versions, so simply strip `const`. - ;; TODO: Consider whether to keep this in 2.0. - ((eventually-call? (cadr arg)) - (expand-forms arg)) - ((and (pair? (cadr arg)) (eq? (caadr arg) 'curly)) - (expand-unionall-def (cadr arg) (caddr arg))) - ((and (pair? (cadr arg)) (eq? (caadr arg) 'tuple) (not (has-parameters? (cdr (cadr arg))))) - ;; We need this case because `(f(), g()) = (1, 2)` goes through here, which cannot go via the `local` lowering below, - ;; because the symbols come out wrong. Sigh... So much effort for such a syntax corner case. - (expand-tuple-destruct (cdr (cadr arg)) (caddr arg) (lambda (assgn) `(,(car e) ,assgn)))) - (else - (let ((rr (make-ssavalue))) - (expand-forms `(block - (= ,rr ,(caddr arg)) - (scope-block (block (hardscope) - (local (= ,(cadr arg) ,rr)) - ,.(map (lambda (v) `(,(car e) (globalref (thismodule) ,v) ,v)) (filter-not-underscore (lhs-vars (cadr arg)))) - (latestworld) - ,rr)))))))) - (else (error "expected assignment after \"const\""))))))) +(define (expand-const-decl e) + (define (check-assignment asgn) + (unless (and (pair? asgn) (eq? (car asgn) '=)) + ;; (const (global x)) is possible due to a parser quirk + (error "expected assignment after \"const\""))) + (if (length= e 3) + `(const ,(cadr e) ,(expand-forms (caddr e))) + (let ((arg (cadr e))) + (case (car arg) + ((global) (let ((asgn (cadr arg))) + (check-assignment asgn) + `(block + ,.(map (lambda (v) `(global ,v)) + (lhs-bound-names (cadr asgn))) + ,(expand-assignment asgn #t)))) + ((=) (check-assignment arg) + (expand-assignment arg #t)) + (else (error "expected assignment after \"const\"")))))) (define (expand-atomic-decl e) (error "unimplemented or unsupported atomic declaration")) @@ -1527,6 +1513,152 @@ (else (error (string "invalid syntax in \"" what "\" declaration")))))))) +(define (expand-assignment e (const? #f)) + (define lhs (cadr e)) + (define (function-lhs? lhs) + (and (pair? lhs) + (or (eq? (car lhs) 'call) + (eq? (car lhs) 'where) + (and (eq? (car lhs) '|::|) + (pair? (cadr lhs)) + (eq? (car (cadr lhs)) 'call))))) + (define (assignment-to-function lhs e) ;; convert '= expr to 'function expr + (cons 'function (cdr e))) + (define (maybe-wrap-const x) + (if const? `(const ,x) x)) + (cond + ((function-lhs? lhs) + ;; `const f() = ...` - The `const` here is inoperative, but the syntax + ;; happened to work in earlier versions, so simply strip `const`. + (expand-forms (assignment-to-function lhs e))) + ((and (pair? lhs) + (eq? (car lhs) 'curly)) + (expand-unionall-def (cadr e) (caddr e) const?)) + ((assignment? (caddr e)) + ;; chain of assignments - convert a=b=c to `b=c; a=c` + (let loop ((lhss (list lhs)) + (rhs (caddr e))) + (if (and (assignment? rhs) (not (function-lhs? (cadr rhs)))) + (loop (cons (cadr rhs) lhss) (caddr rhs)) + (let* ((rr (if (symbol-like? rhs) rhs (make-ssavalue))) + (lhss (reverse lhss)) + (lhs0 (car lhss)) + (lhss (cdr lhss)) + (lhss (reverse lhss))) + (expand-forms + `(block ,.(if (eq? rr rhs) '() `((= ,rr ,(if (assignment? rhs) + (assignment-to-function (cadr rhs) rhs) + rhs)))) + ,@(map (lambda (l) `(= ,l ,rr)) lhss) + ;; In const x = y = z, only x becomes const + ,(maybe-wrap-const `(= ,lhs0 ,rr)) + (unnecessary ,rr))))))) + ((or (and (symbol-like? lhs) (valid-name? lhs)) + (globalref? lhs)) + ;; TODO: We currently call (latestworld) after every (const _ _), but this + ;; may need to be moved elsewhere if we want to avoid making one const + ;; visible before side effects have been performed (#57484) + (if const? + (let ((rr (make-ssavalue))) + `(block + ,(sink-assignment rr (expand-forms (caddr e))) + (const ,lhs ,rr) + (latestworld) + (unnecessary ,rr))) + (sink-assignment lhs (expand-forms (caddr e))))) + ((atom? lhs) + (error (string "invalid assignment location \"" (deparse lhs) "\""))) + (else + (case (car lhs) + ((|.|) + ;; a.b = + (when const? + (error (string "cannot declare \"" (deparse lhs) "\" `const`"))) + (let* ((a (cadr lhs)) + (b (caddr lhs)) + (rhs (caddr e))) + (if (and (length= b 2) (eq? (car b) 'tuple)) + (error (string "invalid syntax \"" + (string (deparse a) ".(" (deparse (cadr b)) ") = ...") "\""))) + (let ((aa (if (symbol-like? a) a (make-ssavalue))) + (bb (if (or (atom? b) (symbol-like? b) (and (pair? b) (quoted? b))) + b (make-ssavalue))) + (rr (if (or (symbol-like? rhs) (atom? rhs)) rhs (make-ssavalue)))) + `(block + ,.(if (eq? aa a) '() (list (sink-assignment aa (expand-forms a)))) + ,.(if (eq? bb b) '() (list (sink-assignment bb (expand-forms b)))) + ,.(if (eq? rr rhs) '() (list (sink-assignment rr (expand-forms rhs)))) + (call (top setproperty!) ,aa ,bb ,rr) + (unnecessary ,rr))))) + ((tuple) + (let ((lhss (cdr lhs)) + (x (caddr e))) + (if (has-parameters? lhss) + ;; property destructuring + (expand-property-destruct lhss x maybe-wrap-const) + ;; multiple assignment + (expand-tuple-destruct lhss x maybe-wrap-const)))) + ((typed_hcat) + (error "invalid spacing in left side of indexed assignment")) + ((typed_vcat typed_ncat) + (error "unexpected \";\" in left side of indexed assignment")) + ((ref) + ;; (= (ref a . idxs) rhs) + (when const? + (error (string "cannot declare \"" (deparse lhs) "\" `const`"))) + (let ((a (cadr lhs)) + (idxs (cddr lhs)) + (rhs (caddr e))) + (let* ((reuse (and (pair? a) + (contains (lambda (x) (eq? x 'end)) + idxs))) + (arr (if reuse (make-ssavalue) a)) + (stmts (if reuse `((= ,arr ,(expand-forms a))) '())) + (rrhs (and (pair? rhs) (not (ssavalue? rhs)) (not (quoted? rhs)))) + (r (if rrhs (make-ssavalue) rhs)) + (rini (if rrhs (list (sink-assignment r (expand-forms rhs))) '()))) + (receive + (new-idxs stuff) (process-indices arr idxs) + `(block + ,@stmts + ,.(map expand-forms stuff) + ,@rini + ,(expand-forms + `(call (top setindex!) ,arr ,r ,@new-idxs)) + (unnecessary ,r)))))) + ((|::|) + ;; (= (|::| T) rhs) is an error + (if (null? (cddr lhs)) + (error (string "invalid assignment location \"" (deparse lhs) "\""))) + ;; (= (|::| x T) rhs) + (let ((x (cadr lhs)) + (T (caddr lhs)) + (rhs (caddr e))) + (let ((e (remove-argument-side-effects x))) + (if const? + ;; This could go through convert-assignment in the closure + ;; conversion pass, but since constants don't have declared types + ;; the way other variables do, we insert convert() here. + (expand-forms + ;; TODO: This behaviour (`const _:T = ...` does not call convert, + ;; but still evaluates RHS) should be documented. + `(const ,(car e) ,(if (underscore-symbol? (car e)) + rhs + (convert-for-type-decl rhs T #t #f)))) + (expand-forms + `(block ,@(cdr e) + ;; TODO: When x is a complex expression, this acts as a + ;; typeassert rather than a declaration. + ,.(if (underscore-symbol? (car e)) + '() ; Assignment to _ will ultimately be discarded---don't declare anything + `((decl ,(car e) ,T))) + ,(maybe-wrap-const `(= ,(car e) ,rhs)))))))) + ((vcat ncat) + ;; (= (vcat . args) rhs) + (error "use \"(a, b) = ...\" to assign multiple values")) + (else + (error (string "invalid assignment location \"" (deparse lhs) "\""))))))) + ;; convert (lhss...) = (tuple ...) to assignments, eliminating the tuple (define (tuple-to-assignments lhss0 x wrap) (let loop ((lhss lhss0) @@ -2262,7 +2394,7 @@ (gensy)) (else (make-ssavalue)))) -(define (expand-property-destruct lhs x) +(define (expand-property-destruct lhs x (wrap identity)) (if (not (length= lhs 1)) (error (string "invalid assignment location \"" (deparse `(tuple ,lhs)) "\""))) (let* ((lhss (cdar lhs)) @@ -2277,7 +2409,7 @@ (cadr field)) (else (error (string "invalid assignment location \"" (deparse `(tuple ,lhs)) "\"")))))) - (expand-forms `(= ,field (call (top getproperty) ,xx (quote ,prop)))))) + (expand-forms (wrap `(= ,field (call (top getproperty) ,xx (quote ,prop))))))) lhss) (unnecessary ,xx)))) @@ -2298,7 +2430,6 @@ (if (null? lhss) '() (let* ((lhs (car lhss)) - (wrapfirst (lambda (x i) (if (= i 1) (wrap x) x))) (lhs- (cond ((or (symbol? lhs) (ssavalue? lhs)) lhs) ((vararg? lhs) @@ -2310,7 +2441,10 @@ (make-ssavalue)))))) ;; can't use ssavalues if it's a function definition ((eventually-call? lhs) (gensy)) - (else (make-ssavalue))))) + (else (make-ssavalue)))) + ;; If we use an intermediary lhs, don't wrap `const`. + (wrap-subassign (if (eq? lhs lhs-) wrap identity)) + (wrapfirst (lambda (x i) (if (= i 1) (wrap-subassign x) x)))) (if (and (vararg? lhs) (any vararg? (cdr lhss))) (error "multiple \"...\" on lhs of assignment")) (if (not (eq? lhs lhs-)) @@ -2322,7 +2456,7 @@ (if (underscore-symbol? (cadr lhs-)) '() (list (expand-forms - (wrap `(= ,(cadr lhs-) (call (top rest) ,xx ,@(if (eq? i 1) '() `(,st)))))))) + (wrap-subassign `(= ,(cadr lhs-) (call (top rest) ,xx ,@(if (eq? i 1) '() `(,st)))))))) (let ((tail (if (eventually-call? lhs) (gensy) (make-ssavalue)))) (cons (expand-forms (lower-tuple-assignment @@ -2498,115 +2632,7 @@ 'global expand-local-or-global-decl 'local-def expand-local-or-global-decl - '= - (lambda (e) - (define lhs (cadr e)) - (define (function-lhs? lhs) - (and (pair? lhs) - (or (eq? (car lhs) 'call) - (eq? (car lhs) 'where) - (and (eq? (car lhs) '|::|) - (pair? (cadr lhs)) - (eq? (car (cadr lhs)) 'call))))) - (define (assignment-to-function lhs e) ;; convert '= expr to 'function expr - (cons 'function (cdr e))) - (cond - ((function-lhs? lhs) - (expand-forms (assignment-to-function lhs e))) - ((and (pair? lhs) - (eq? (car lhs) 'curly)) - (expand-unionall-def (cadr e) (caddr e))) - ((assignment? (caddr e)) - ;; chain of assignments - convert a=b=c to `b=c; a=c` - (let loop ((lhss (list lhs)) - (rhs (caddr e))) - (if (and (assignment? rhs) (not (function-lhs? (cadr rhs)))) - (loop (cons (cadr rhs) lhss) (caddr rhs)) - (let ((rr (if (symbol-like? rhs) rhs (make-ssavalue)))) - (expand-forms - `(block ,.(if (eq? rr rhs) '() `((= ,rr ,(if (assignment? rhs) - (assignment-to-function (cadr rhs) rhs) - rhs)))) - ,@(map (lambda (l) `(= ,l ,rr)) - lhss) - (unnecessary ,rr))))))) - ((or (and (symbol-like? lhs) (valid-name? lhs)) - (globalref? lhs)) - (sink-assignment lhs (expand-forms (caddr e)))) - ((atom? lhs) - (error (string "invalid assignment location \"" (deparse lhs) "\""))) - (else - (case (car lhs) - ((|.|) - ;; a.b = - (let* ((a (cadr lhs)) - (b (caddr lhs)) - (rhs (caddr e))) - (if (and (length= b 2) (eq? (car b) 'tuple)) - (error (string "invalid syntax \"" - (string (deparse a) ".(" (deparse (cadr b)) ") = ...") "\""))) - (let ((aa (if (symbol-like? a) a (make-ssavalue))) - (bb (if (or (atom? b) (symbol-like? b) (and (pair? b) (quoted? b))) - b (make-ssavalue))) - (rr (if (or (symbol-like? rhs) (atom? rhs)) rhs (make-ssavalue)))) - `(block - ,.(if (eq? aa a) '() (list (sink-assignment aa (expand-forms a)))) - ,.(if (eq? bb b) '() (list (sink-assignment bb (expand-forms b)))) - ,.(if (eq? rr rhs) '() (list (sink-assignment rr (expand-forms rhs)))) - (call (top setproperty!) ,aa ,bb ,rr) - (unnecessary ,rr))))) - ((tuple) - (let ((lhss (cdr lhs)) - (x (caddr e))) - (if (has-parameters? lhss) - ;; property destructuring - (expand-property-destruct lhss x) - ;; multiple assignment - (expand-tuple-destruct lhss x)))) - ((typed_hcat) - (error "invalid spacing in left side of indexed assignment")) - ((typed_vcat typed_ncat) - (error "unexpected \";\" in left side of indexed assignment")) - ((ref) - ;; (= (ref a . idxs) rhs) - (let ((a (cadr lhs)) - (idxs (cddr lhs)) - (rhs (caddr e))) - (let* ((reuse (and (pair? a) - (contains (lambda (x) (eq? x 'end)) - idxs))) - (arr (if reuse (make-ssavalue) a)) - (stmts (if reuse `((= ,arr ,(expand-forms a))) '())) - (rrhs (and (pair? rhs) (not (ssavalue? rhs)) (not (quoted? rhs)))) - (r (if rrhs (make-ssavalue) rhs)) - (rini (if rrhs (list (sink-assignment r (expand-forms rhs))) '()))) - (receive - (new-idxs stuff) (process-indices arr idxs) - `(block - ,@stmts - ,.(map expand-forms stuff) - ,@rini - ,(expand-forms - `(call (top setindex!) ,arr ,r ,@new-idxs)) - (unnecessary ,r)))))) - ((|::|) - ;; (= (|::| T) rhs) is an error - (if (null? (cddr lhs)) - (error (string "invalid assignment location \"" (deparse lhs) "\""))) - ;; (= (|::| x T) rhs) - (let ((x (cadr lhs)) - (T (caddr lhs)) - (rhs (caddr e))) - (let ((e (remove-argument-side-effects x))) - (expand-forms - `(block ,@(cdr e) - (decl ,(car e) ,T) - (= ,(car e) ,rhs)))))) - ((vcat ncat) - ;; (= (vcat . args) rhs) - (error "use \"(a, b) = ...\" to assign multiple values")) - (else - (error (string "invalid assignment location \"" (deparse lhs) "\""))))))) + '= expand-assignment 'abstract (lambda (e) @@ -2975,6 +3001,16 @@ (define (lhs-vars e) (map decl-var (lhs-decls e))) +;; Return all the names that will be bound by the assignment LHS, including +;; curlies and calls. +(define (lhs-bound-names e) + (cond ((underscore-symbol? e) '()) + ((atom? e) (list e)) + ((and (pair? e) (memq (car e) '(call curly where |::|))) + (lhs-bound-names (cadr e))) + ((and (pair? e) (memq (car e) '(tuple parameters))) + (apply append (map lhs-bound-names (cdr e)))))) + (define (all-decl-vars e) ;; map decl-var over every level of an assignment LHS (cond ((eventually-call? e) e) ((decl? e) (decl-var e)) @@ -3001,7 +3037,7 @@ ;; like v = val, except that if `v` turns out global(either ;; implicitly or by explicit `global`), it gains an implicit `const` (set! vars (cons (cadr e) vars))) - ((=) + ((= const) (let ((v (decl-var (cadr e)))) (find-assigned-vars- (caddr e)) (if (or (ssavalue? v) (globalref? v) (underscore-symbol? v)) @@ -3127,14 +3163,18 @@ ((eq? (car e) 'global) (check-valid-name (cadr e)) e) + ((eq? (car e) 'assign-const-if-global) (if (eq? (var-kind (cadr e) scope) 'local) - (if (length= e 2) (null) `(= ,@(cdr e))) - `(const ,@(cdr e)))) + (if (length= e 2) + (null) + (resolve-scopes- `(= ,@(cdr e)) scope sp loc)) + (resolve-scopes- `(const ,@(cdr e)) scope sp loc))) ((eq? (car e) 'global-if-global) (if (eq? (var-kind (cadr e) scope) 'local) '(null) `(global ,@(cdr e)))) + ((memq (car e) '(local local-def)) (check-valid-name (cadr e)) ;; remove local decls @@ -3287,7 +3327,7 @@ ,(resolve-scopes- (caddr e) scope) ,(resolve-scopes- (cadddr e) scope (method-expr-static-parameters e)))) (else - (if (and (eq? (car e) '=) (symbol? (cadr e)) + (if (and (memq (car e) '(= const)) (symbol? (cadr e)) scope (null? (lam:args (scope:lam scope))) (warn-var?! (cadr e) scope) (= *scopewarn-opt* 1)) @@ -3407,7 +3447,7 @@ ((local-def) ;; a local that we know has an assignment that dominates all usages (let ((vi (get tab (cadr e) #f))) (vinfo:set-never-undef! vi #t))) - ((=) + ((= const) (let ((vi (and (symbol? (cadr e)) (get tab (cadr e) #f)))) (if vi ; if local or captured (begin (if (vinfo:asgn vi) @@ -4024,7 +4064,10 @@ f(x) = yt(x) '(null) `(newvar ,(cadr e)))))) ((const) - (put! globals (binding-to-globalref (cadr e)) #f) + ;; Check we've expanded surface `const` (1 argument form) + (assert (and (length= e 3))) + (when (globalref? (cadr e)) + (put! globals (cadr e) #f)) e) ((atomic) e) ((isdefined) ;; convert isdefined expr to function for closure converted variables @@ -4376,7 +4419,6 @@ f(x) = yt(x) (first-line #t) (current-loc #f) (rett #f) - (global-const-error #f) (vinfo-table (vinfo-to-table (car (lam:vinfo lam)))) (arg-map #f) ;; map arguments to new names if they are assigned (label-counter 0) ;; counter for generating label addresses @@ -4589,18 +4631,19 @@ f(x) = yt(x) (cdr cnd) (list cnd)))))) tests)) - (define (emit-assignment-or-setglobal lhs rhs) - (if (globalref? lhs) + (define (emit-assignment-or-setglobal lhs rhs (op '=)) + ;; (const (globalref _ _) _) does not use setglobal! + (if (and (globalref? lhs) (eq? op '=)) (emit `(call (top setglobal!) ,(cadr lhs) (inert ,(caddr lhs)) ,rhs)) - (emit `(= ,lhs ,rhs)))) - (define (emit-assignment lhs rhs) + (emit `(,op ,lhs ,rhs)))) + (define (emit-assignment lhs rhs (op '=)) (if rhs (if (valid-ir-rvalue? lhs rhs) - (emit-assignment-or-setglobal lhs rhs) + (emit-assignment-or-setglobal lhs rhs op) (let ((rr (make-ssavalue))) (emit `(= ,rr ,rhs)) - (emit-assignment-or-setglobal lhs rr))) - (emit-assignment-or-setglobal lhs `(null))) ; in unreachable code (such as after return), still emit the assignment so that the structure of those uses is preserved + (emit-assignment-or-setglobal lhs rr op))) + (emit-assignment-or-setglobal lhs `(null) op)) ; in unreachable code (such as after return), still emit the assignment so that the structure of those uses is preserved #f) ;; the interpreter loop. `break-labels` keeps track of the labels to jump to ;; for all currently closing break-blocks. @@ -4666,7 +4709,12 @@ f(x) = yt(x) (cond (tail (emit-return tail callex)) (value callex) (else (emit callex))))) - ((=) + ((= const) + (when (eq? (car e) 'const) + (when (local-in? (cadr e) lam) + (error (string "unsupported `const` declaration on local variable" (format-loc current-loc)))) + (when (pair? (cadr lam)) + (error (string "`global const` declaration not allowed inside function" (format-loc current-loc))))) (let ((lhs (cadr e))) (if (and (symbol? lhs) (underscore-symbol? lhs)) (compile (caddr e) break-labels value tail) @@ -4679,10 +4727,10 @@ f(x) = yt(x) rhs (make-ssavalue)))) (if (not (eq? rr rhs)) (emit `(= ,rr ,rhs))) - (emit-assignment-or-setglobal lhs rr) + (emit-assignment-or-setglobal lhs rr (car e)) (if tail (emit-return tail rr)) rr) - (emit-assignment lhs rhs)))))) + (emit-assignment lhs rhs (car e))))))) ((block) (let* ((last-fname filename) (fnm (first-non-meta e)) @@ -4925,14 +4973,6 @@ f(x) = yt(x) ((moved-local) (set-car! (lam:vinfo lam) (append (car (lam:vinfo lam)) `((,(cadr e) Any 2)))) #f) - ((const) - (if (local-in? (cadr e) lam) - (error (string "unsupported `const` declaration on local variable" (format-loc current-loc))) - (if (pair? (cadr lam)) - ;; delay this error to allow "misplaced struct" errors to happen first - (if (not global-const-error) - (set! global-const-error current-loc)) - (emit e)))) ((atomic) (error "misplaced atomic declaration")) ((isdefined throw_undef_if_not) (if tail (emit-return tail e) e)) ((boundscheck) (if tail (emit-return tail e) e)) @@ -5063,8 +5103,6 @@ f(x) = yt(x) (let ((pexc (pop-exc-expr src-catch-tokens target-catch-tokens))) (if pexc (set-cdr! point (cons pexc (cdr point))))))))) handler-goto-fixups) - (if global-const-error - (error (string "`global const` declaration not allowed inside function" (format-loc global-const-error)))) (let* ((stmts (reverse! code)) (di (definitely-initialized-vars stmts vi)) (body (cons 'block (filter (lambda (e) diff --git a/test/syntax.jl b/test/syntax.jl index dc269da0475de..7e09f4747f939 100644 --- a/test/syntax.jl +++ b/test/syntax.jl @@ -3816,7 +3816,7 @@ module ImplicitCurlies end @test !@isdefined(ImplicitCurly6) # Check return value of assignment expr - @test isa((const ImplicitCurly7{T} = Ref{T}), UnionAll) + @test isa(Core.eval(@__MODULE__, :(const ImplicitCurly7{T} = Ref{T})), UnionAll) @test isa(begin; ImplicitCurly8{T} = Ref{T}; end, UnionAll) end @@ -3852,7 +3852,8 @@ const (gconst_assign(), hconst_assign()) = (2, 3) # and the conversion, but not the rhs. struct CantConvert; end Base.convert(::Type{CantConvert}, x) = error() -@test (const _::CantConvert = 1) == 1 +# @test splices into a function, where const cannot appear +@test Core.eval(@__MODULE__, :(const _::CantConvert = 1)) == 1 @test !isconst(@__MODULE__, :_) @test_throws ErrorException("expected") (const _ = error("expected")) @@ -4136,3 +4137,102 @@ module FuncDecl57546 @test isa(Any, Function) @test isempty(methods(Any)) end + +# #57334 +let + x57334 = Ref(1) + @test_throws "syntax: cannot declare \"x57334[]\" `const`" Core.eval(@__MODULE__, :(const x57334[] = 1)) +end + +# #57470 +module M57470 +using ..Test + +@test_throws( + "syntax: `global const` declaration not allowed inside function", + Core.eval(@__MODULE__, :(function f57470() + const global x57470 = 1 + end))) +@test_throws( + "unsupported `const` declaration on local variable", + Core.eval(@__MODULE__, :(let + const y57470 = 1 + end)) +) + +let + global a57470 + const a57470 = 1 +end +@test a57470 === 1 + +let + global const z57470 = 1 + const global w57470 = 1 +end + +@test z57470 === 1 +@test w57470 === 1 + +const (; field57470_1, field57470_2) = (field57470_1 = 1, field57470_2 = 2) +@test field57470_1 === 1 +@test field57470_2 === 2 + +# TODO: 1.11 allows these, but should we? +const X57470{T}, Y57470{T} = Int, Bool +@test X57470 === Int +@test Y57470 === Bool +const A57470{T}, B57470{T} = [Int, Bool] +@test A57470 === Int +@test B57470 === Bool +const a57470, f57470(x), T57470{U} = [1, 2, Int] +@test a57470 === 1 +@test f57470(0) === 2 +@test T57470 === Int + +module M57470_sub end +@test_throws("syntax: cannot declare \"M57470_sub.x\" `const`", + Core.eval(@__MODULE__, :(const M57470_sub.x = 1))) + +# # `const global` should not trample previously declared `local` +@test_throws( + "syntax: variable \"v57470\" declared both local and global", + Core.eval(@__MODULE__, :(let + local v57470 + const global v57470 = 1 + end)) +) + +# Chain of assignments must happen right-to-left: +let + x = [0, 0]; i = 1 + i = x[i] = 2 + @test x == [2, 0] + x = [0, 0]; i = 1 + x[i] = i = 2 + @test x == [0, 2] +end + +# Global const decl inside local scope +let + const global letf_57470(x)::Int = 2+x + const global letT_57470{T} = Int64 +end +@test letf_57470(3) == 5 +@test letT_57470 === Int64 + +end + +# #57574 +module M57574 +struct A{T} end +out = let + for B in () + end + let + B{T} = A{T} + B + end +end +end +@test M57574.out === M57574.A From 02e5c8304b0b7cc60fb6e22d46ef5041baee6e49 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 20 Mar 2025 16:01:29 -0500 Subject: [PATCH 59/74] Fix package precompilation (PrecompileTools) (#57828) With the "classic" inference timing disabled, PrecompileTools and SnoopCompile are non-functional on 1.12 & master. #57074 added timing data to the CodeInstances themselves, which should help restore SnoopCompile. However, without the tree structure provided by the old inference timing system, we still need a way to tag MethodInstances that get inferred inside `PrecompileTools.@compile_workload` blocks. This adds a simple mechanism modeled after `@trace_compile` that switches to tagging MethodInstances in `jl_push_newly_inferred`. https://github.com/JuliaLang/PrecompileTools.jl/pull/47 contains (or will contain) the corresponding changes to PrecompileTools.jl. (cherry picked from commit c89b1ff7a1206e60e44a2e949041c33c4e853b16) --- src/julia_internal.h | 3 +++ src/staticdata_utils.c | 23 +++++++++++++++++++++++ test/precompile.jl | 40 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/julia_internal.h b/src/julia_internal.h index f358d30b9c193..5f29e9462b816 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1280,6 +1280,9 @@ JL_DLLEXPORT void jl_force_trace_compile_timing_disable(void); JL_DLLEXPORT void jl_force_trace_dispatch_enable(void); JL_DLLEXPORT void jl_force_trace_dispatch_disable(void); +JL_DLLEXPORT void jl_tag_newly_inferred_enable(void); +JL_DLLEXPORT void jl_tag_newly_inferred_disable(void); + uint32_t jl_module_next_counter(jl_module_t *m) JL_NOTSAFEPOINT; jl_tupletype_t *arg_type_tuple(jl_value_t *arg1, jl_value_t **args, size_t nargs); diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 163eb1ea9cad5..e9f464b64470e 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -86,6 +86,23 @@ static jl_array_t *newly_inferred JL_GLOBALLY_ROOTED /*FIXME*/; // Mutex for newly_inferred jl_mutex_t newly_inferred_mutex; extern jl_mutex_t world_counter_lock; +static _Atomic(uint8_t) jl_tag_newly_inferred_enabled = 0; + +/** + * @brief Enable tagging of all newly inferred CodeInstances. + */ +JL_DLLEXPORT void jl_tag_newly_inferred_enable(void) +{ + jl_atomic_fetch_add(&jl_tag_newly_inferred_enabled, 1); // FIXME overflow? +} +/** + * @brief Disable tagging of all newly inferred CodeInstances. + */ +JL_DLLEXPORT void jl_tag_newly_inferred_disable(void) +{ + jl_atomic_fetch_add(&jl_tag_newly_inferred_enabled, -1); // FIXME underflow? +} + // Register array of newly-inferred MethodInstances // This gets called as the first step of Base.include_package_for_output @@ -101,6 +118,12 @@ JL_DLLEXPORT void jl_push_newly_inferred(jl_value_t* ci) { if (!newly_inferred) return; + uint8_t tag_newly_inferred = jl_atomic_load_relaxed(&jl_tag_newly_inferred_enabled); + if (tag_newly_inferred) { + jl_method_instance_t *mi = jl_get_ci_mi((jl_code_instance_t*)ci); + uint8_t miflags = jl_atomic_load_relaxed(&mi->flags); + jl_atomic_store_relaxed(&mi->flags, miflags | JL_MI_FLAGS_MASK_PRECOMPILED); + } JL_LOCK(&newly_inferred_mutex); size_t end = jl_array_nrows(newly_inferred); jl_array_grow_end(newly_inferred, 1); diff --git a/test/precompile.jl b/test/precompile.jl index 9fd588cc50808..e246133abf106 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -747,7 +747,6 @@ end # method root provenance & external code caching precompile_test_harness("code caching") do dir - Bid = rootid(Base) Cache_module = :Cacheb8321416e8a3e2f1 # Note: calling setindex!(::Dict{K,V}, ::Any, ::K) adds both compression and codegen roots write(joinpath(dir, "$Cache_module.jl"), @@ -1077,6 +1076,45 @@ precompile_test_harness("code caching") do dir end end +precompile_test_harness("precompiletools") do dir + PrecompileToolsModule = :PCTb8321416e8a3e2f1 + write(joinpath(dir, "$PrecompileToolsModule.jl"), + """ + module $PrecompileToolsModule + struct MyType + x::Int + end + + function call_findfirst(x, list) + # call a method defined in Base by runtime dispatch + return findfirst(==(Base.inferencebarrier(x)), Base.inferencebarrier(list)) + end + + let + ccall(:jl_tag_newly_inferred_enable, Cvoid, ()) + call_findfirst(MyType(2), [MyType(1), MyType(2), MyType(3)]) + ccall(:jl_tag_newly_inferred_disable, Cvoid, ()) + end + end + """ + ) + pkgid = Base.PkgId(string(PrecompileToolsModule)) + @test !Base.isprecompiled(pkgid) + Base.compilecache(pkgid) + @test Base.isprecompiled(pkgid) + @eval using $PrecompileToolsModule + M = invokelatest(getfield, @__MODULE__, PrecompileToolsModule) + invokelatest() do + m = which(Tuple{typeof(findfirst), Base.Fix2{typeof(==), T}, Vector{T}} where T) + success = 0 + for mi in Base.specializations(m) + sig = Base.unwrap_unionall(mi.specTypes) + success += sig.parameters[3] === Vector{M.MyType} + end + @test success == 1 + end +end + precompile_test_harness("invoke") do dir InvokeModule = :Invoke0x030e7e97c2365aad CallerModule = :Caller0x030e7e97c2365aad From ef270a424921385950a6e9440a388affa6767a04 Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Thu, 20 Mar 2025 22:02:43 +0100 Subject: [PATCH 60/74] restrict dispatch of some custrom string macros to `String` (#57781) Restricts the dispatch for these macros so that only `String` (literal) arguments are accepted: * `int128_str` * `uint128_str` * `big_str` * `ip_str` from the Sockets stdlib * `dateformat_str` from the Dates stdlib Some of these changes make the code in the sysimage less vulnerable to invalidation. (cherry picked from commit f9365a5ecd336388c0d4f3c987c1c8fde95b4da3) --- base/int.jl | 6 +++--- stdlib/Dates/src/io.jl | 2 +- stdlib/Sockets/src/IPAddr.jl | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/base/int.jl b/base/int.jl index a0ab68436145d..9bd4df6bd3866 100644 --- a/base/int.jl +++ b/base/int.jl @@ -642,7 +642,7 @@ ERROR: LoadError: ArgumentError: invalid base 10 digit '.' in "123456789123.4" [...] ``` """ -macro int128_str(s) +macro int128_str(s::String) return parse(Int128, s) end @@ -662,7 +662,7 @@ ERROR: LoadError: ArgumentError: invalid base 10 digit '-' in "-123456789123" [...] ``` """ -macro uint128_str(s) +macro uint128_str(s::String) return parse(UInt128, s) end @@ -695,7 +695,7 @@ ERROR: ArgumentError: invalid number format _ for BigInt or BigFloat depends on the value of the precision at the point when the function is defined, **not** at the precision at the time when the function is called. """ -macro big_str(s) +macro big_str(s::String) message = "invalid number format $s for BigInt or BigFloat" throw_error = :(throw(ArgumentError($message))) if '_' in s diff --git a/stdlib/Dates/src/io.jl b/stdlib/Dates/src/io.jl index aa7019566093c..88c32bf064bf0 100644 --- a/stdlib/Dates/src/io.jl +++ b/stdlib/Dates/src/io.jl @@ -478,7 +478,7 @@ but creates the DateFormat object once during macro expansion. See [`DateFormat`](@ref) for details about format specifiers. """ -macro dateformat_str(str) +macro dateformat_str(str::String) DateFormat(str) end diff --git a/stdlib/Sockets/src/IPAddr.jl b/stdlib/Sockets/src/IPAddr.jl index d3834a8b8bf73..e324dee712b71 100644 --- a/stdlib/Sockets/src/IPAddr.jl +++ b/stdlib/Sockets/src/IPAddr.jl @@ -286,7 +286,7 @@ julia> @ip_str "2001:db8:0:0:0:0:2:1" ip"2001:db8::2:1" ``` """ -macro ip_str(str) +macro ip_str(str::String) return parse(IPAddr, str) end From 22947227cf9e5226a04cdfc924d5dae4eebad980 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Wed, 12 Feb 2025 18:23:41 -0300 Subject: [PATCH 61/74] Respect JULIA_CPU_TARGET when doing trimming (#57373) We could make this have a nicer API but for now lets just do what we've done for pkgimages/sysimages and use the envar (cherry picked from commit 17fff87f583c2fd0bbb85b63d76e5b9be634f2bd) --- contrib/juliac.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/juliac.jl b/contrib/juliac.jl index 8b413fccd6231..6639eb2871d2d 100644 --- a/contrib/juliac.jl +++ b/contrib/juliac.jl @@ -6,6 +6,8 @@ module JuliaConfig end julia_cmd = `$(Base.julia_cmd()) --startup-file=no --history-file=no` +cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) +julia_cmd_target = `$(Base.julia_cmd(;cpu_target)) --startup-file=no --history-file=no` output_type = nothing # exe, sharedlib, sysimage outname = nothing file = nothing @@ -102,7 +104,7 @@ end function compile_products() # Compile the Julia code - cmd = addenv(`$julia_cmd --project=$(Base.active_project()) --output-o $img_path --output-incremental=no --strip-ir --strip-metadata $julia_args $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) + cmd = addenv(`$julia_cmd_target --project=$(Base.active_project()) --output-o $img_path --output-incremental=no --strip-ir --strip-metadata $julia_args $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) verbose && println("Running: $cmd") if !success(pipeline(cmd; stdout, stderr)) println(stderr, "\nFailed to compile $file") From 6f34f9191381107b7c9f6edefc91b7b88e8a6d0a Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Fri, 7 Mar 2025 08:17:42 -0500 Subject: [PATCH 62/74] trimming: Avoid `--strip-ir` when compiling with `--trim=no` (#57659) Both `--trim=no` and `--trim=safe` are supposed to be safe options by default, so let's not get overzealous and throw out all the IR even though we're not trimming Allows `juliac.jl` to use/test the sysimage creation pipeline unrelated to `--trim` support, as in https://github.com/JuliaLang/julia/pull/57656#discussion_r1983782350 (cherry picked from commit f9e5af1464891e107eb0dc487a4afd7da1323b0f) --- contrib/juliac.jl | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/contrib/juliac.jl b/contrib/juliac.jl index 6639eb2871d2d..7087462afc7a1 100644 --- a/contrib/juliac.jl +++ b/contrib/juliac.jl @@ -30,6 +30,7 @@ end # arguments to forward to julia compilation process julia_args = [] +enable_trim::Bool = false let i = 1 while i <= length(ARGS) @@ -46,9 +47,11 @@ let i = 1 global verbose = true elseif arg == "--relative-rpath" global relative_rpath = true - elseif startswith(arg, "--trim") || arg == "--experimental" - # forwarded args - push!(julia_args, arg) + elseif startswith(arg, "--trim") + global enable_trim = arg != "--trim=no" + push!(julia_args, arg) # forwarded arg + elseif arg == "--experimental" + push!(julia_args, arg) # forwarded arg else if arg[1] == '-' || !isnothing(file) println("Unexpected argument `$arg`") @@ -102,9 +105,17 @@ function precompile_env() end end -function compile_products() +function compile_products(enable_trim::Bool) + + # Only strip IR / metadata if not `--trim=no` + strip_args = String[] + if enable_trim + push!(strip_args, "--strip-ir") + push!(strip_args, "--strip-metadata") + end + # Compile the Julia code - cmd = addenv(`$julia_cmd_target --project=$(Base.active_project()) --output-o $img_path --output-incremental=no --strip-ir --strip-metadata $julia_args $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) + cmd = addenv(`$julia_cmd_target --project=$(Base.active_project()) --output-o $img_path --output-incremental=no $strip_args $julia_args $(joinpath(@__DIR__,"juliac-buildscript.jl")) $absfile $output_type $add_ccallables`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1) verbose && println("Running: $cmd") if !success(pipeline(cmd; stdout, stderr)) println(stderr, "\nFailed to compile $file") @@ -154,5 +165,5 @@ function link_products() end precompile_env() -compile_products() +compile_products(enable_trim) link_products() From e342f1c14d24a82229332c63be6c13b9712d4ebf Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Thu, 6 Mar 2025 21:41:47 -0500 Subject: [PATCH 63/74] trimming: Avoid `using` indirectly-loaded packages (#57589) Although a package is loaded, we may not have top-level `using` rights. Instead, we need to pull the Module straight out of the loaded modules array in order to do these hacks. (cherry picked from commit 1cead2b917db6f2ff8345470cb7c313f46b5c31d) --- contrib/juliac-buildscript.jl | 36 ++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index 6be2d20597079..98990ed03306a 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -208,10 +208,15 @@ let mod = Base.include(Base.__toplevel__, inputfile) end # Additional method patches depending on whether user code loads certain stdlibs +let loaded = Base.loaded_modules_array() + function find_loaded_module(name) + idx = findfirst((m) -> Symbol(m) === name, loaded) + idx === nothing && return nothing + return loaded[idx] + end -let loaded = Symbol.(Base.loaded_modules_array()) # TODO better way to do this - if :SparseArrays in loaded - using SparseArrays + SparseArrays = find_loaded_module(:SparseArrays) + if SparseArrays !== nothing @eval SparseArrays.CHOLMOD begin function __init__() ccall((:SuiteSparse_config_malloc_func_set, :libsuitesparseconfig), @@ -225,8 +230,9 @@ let loaded = Symbol.(Base.loaded_modules_array()) # TODO better way to do this end end end - if :Artifacts in loaded - using Artifacts + + Artifacts = find_loaded_module(:Artifacts) + if Artifacts !== nothing @eval Artifacts begin function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, _::Val{lazyartifacts}) where lazyartifacts # If the artifact exists, we're in the happy path and we can immediately @@ -241,26 +247,30 @@ let loaded = Symbol.(Base.loaded_modules_array()) # TODO better way to do this end end end - if :Pkg in loaded - using Pkg + + Pkg = find_loaded_module(:Pkg) + if Pkg !== nothing @eval Pkg begin __init__() = rand() #TODO, methods that do nothing don't get codegened end end - if :StyledStrings in loaded - using StyledStrings + + StyledStrings = find_loaded_module(:StyledStrings) + if StyledStrings !== nothing @eval StyledStrings begin __init__() = rand() end end - if :Markdown in loaded - using Markdown + + Markdown = find_loaded_module(:Markdown) + if Markdown !== nothing @eval Markdown begin __init__() = rand() end end - if :JuliaSyntaxHighlighting in loaded - using JuliaSyntaxHighlighting + + JuliaSyntaxHighlighting = find_loaded_module(:JuliaSyntaxHighlighting) + if JuliaSyntaxHighlighting !== nothing @eval JuliaSyntaxHighlighting begin __init__() = rand() end From 4652a0925af8824d8f7144d88234ca103323de67 Mon Sep 17 00:00:00 2001 From: Cody Tapscott <84105208+topolarity@users.noreply.github.com> Date: Sat, 8 Mar 2025 09:52:50 -0500 Subject: [PATCH 64/74] trimming: Locate package modules by PkgId (#57672) As recommended in https://github.com/JuliaLang/julia/pull/57589#discussion_r1984563023 (cherry picked from commit eba2a337f914462b47a26fb30939155beab72ace) --- contrib/juliac-buildscript.jl | 37 +++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index 98990ed03306a..f51238cc0862a 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -208,14 +208,11 @@ let mod = Base.include(Base.__toplevel__, inputfile) end # Additional method patches depending on whether user code loads certain stdlibs -let loaded = Base.loaded_modules_array() - function find_loaded_module(name) - idx = findfirst((m) -> Symbol(m) === name, loaded) - idx === nothing && return nothing - return loaded[idx] - end +let + find_loaded_root_module(key::Base.PkgId) = Base.maybe_root_module(key) - SparseArrays = find_loaded_module(:SparseArrays) + SparseArrays = find_loaded_root_module(Base.PkgId( + Base.UUID("2f01184e-e22b-5df5-ae63-d93ebab69eaf"), "SparseArrays")) if SparseArrays !== nothing @eval SparseArrays.CHOLMOD begin function __init__() @@ -231,10 +228,20 @@ let loaded = Base.loaded_modules_array() end end - Artifacts = find_loaded_module(:Artifacts) + Artifacts = find_loaded_root_module(Base.PkgId( + Base.UUID("56f22d72-fd6d-98f1-02f0-08ddc0907c33"), "Artifacts")) if Artifacts !== nothing @eval Artifacts begin - function _artifact_str(__module__, artifacts_toml, name, path_tail, artifact_dict, hash, platform, _::Val{lazyartifacts}) where lazyartifacts + function _artifact_str( + __module__, + artifacts_toml, + name, + path_tail, + artifact_dict, + hash, + platform, + _::Val{LazyArtifacts} + ) where LazyArtifacts # If the artifact exists, we're in the happy path and we can immediately # return the path to the artifact: dirs = artifacts_dirs(bytes2hex(hash.bytes)) @@ -248,28 +255,32 @@ let loaded = Base.loaded_modules_array() end end - Pkg = find_loaded_module(:Pkg) + Pkg = find_loaded_root_module(Base.PkgId( + Base.UUID("44cfe95a-1eb2-52ea-b672-e2afdf69b78f"), "Pkg")) if Pkg !== nothing @eval Pkg begin __init__() = rand() #TODO, methods that do nothing don't get codegened end end - StyledStrings = find_loaded_module(:StyledStrings) + StyledStrings = find_loaded_root_module(Base.PkgId( + Base.UUID("f489334b-da3d-4c2e-b8f0-e476e12c162b"), "StyledStrings")) if StyledStrings !== nothing @eval StyledStrings begin __init__() = rand() end end - Markdown = find_loaded_module(:Markdown) + Markdown = find_loaded_root_module(Base.PkgId( + Base.UUID("d6f4376e-aef5-505a-96c1-9c027394607a"), "Markdown")) if Markdown !== nothing @eval Markdown begin __init__() = rand() end end - JuliaSyntaxHighlighting = find_loaded_module(:JuliaSyntaxHighlighting) + JuliaSyntaxHighlighting = find_loaded_root_module(Base.PkgId( + Base.UUID("ac6e5ff7-fb65-4e79-a425-ec3bc9c03011"), "JuliaSyntaxHighlighting")) if JuliaSyntaxHighlighting !== nothing @eval JuliaSyntaxHighlighting begin __init__() = rand() From 5e8ac8527f380fb0eba9269067eab50d5b1b4a99 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Sun, 16 Mar 2025 23:02:53 -0400 Subject: [PATCH 65/74] [internals] add time metrics for every CodeInstance (#57074) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds 4 new Float16 fields to CodeInstance to replace Compiler.Timings with continually collected and available measurements. Sample results on a novel method signature: julia> @time code_native(devnull, ÷, dump_module=false, (Int32, UInt16)); 0.006262 seconds (3.62 k allocations: 186.641 KiB, 75.53% compilation time) julia> b = which(÷, (Int32, UInt16)).specializations[6].cache CodeInstance for MethodInstance for div(::Int32, ::UInt16) julia> reinterpret(Float16, b.time_infer_self) Float16(0.0002766) julia> reinterpret(Float16, b.time_infer_total) Float16(0.00049) julia> reinterpret(Float16, b.time_infer_cache_saved) Float16(0.02774) julia> reinterpret(Float16, b.time_compile) Float16(0.003773) Closes https://github.com/JuliaLang/julia/issues/56115 (cherry picked from commit 18b5d8fb8d72a43cc5521ec0ebcd8de503cf885c) --- Compiler/src/abstractinterpretation.jl | 19 +++++- Compiler/src/inferencestate.jl | 10 ++- Compiler/src/typeinfer.jl | 54 +++++++++++----- Compiler/src/utilities.jl | 2 + base/Base.jl | 2 - base/Base_compiler.jl | 2 + base/essentials.jl | 2 +- base/float.jl | 87 ++------------------------ base/hashing.jl | 81 ++++++++++++++++++++++++ base/rounding.jl | 2 +- doc/src/devdocs/ast.md | 20 ++++++ src/gf.c | 7 +++ src/jitlayers.cpp | 5 +- src/jltypes.c | 28 ++++++--- src/julia.h | 9 ++- src/julia_internal.h | 11 ++-- src/staticdata.c | 1 + test/core.jl | 4 +- 18 files changed, 221 insertions(+), 125 deletions(-) diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index b29b65b737382..c0b079d700bee 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -1352,6 +1352,8 @@ function const_prop_call(interp::AbstractInterpreter, end assign_parentchild!(frame, sv) if !typeinf(interp, frame) + sv.time_caches += frame.time_caches + sv.time_paused += frame.time_paused add_remark!(interp, sv, "[constprop] Fresh constant inference hit a cycle") @assert frame.frameid != 0 && frame.cycleid == frame.frameid callstack = frame.callstack::Vector{AbsIntState} @@ -4313,6 +4315,7 @@ end # make as much progress on `frame` as possible (by handling cycles) warnlength::Int = 2500 function typeinf(interp::AbstractInterpreter, frame::InferenceState) + time_before = _time_ns() callstack = frame.callstack::Vector{AbsIntState} nextstates = CurrentState[] takenext = frame.frameid @@ -4344,7 +4347,6 @@ function typeinf(interp::AbstractInterpreter, frame::InferenceState) # get_compileable_sig), but still must be finished up since it may see and # change the local variables of the InferenceState at currpc, we do this # even if the nextresult status is already completed. - continue elseif isdefined(nextstates[nextstateid], :result) || !isempty(callee.ip) # Next make progress on this frame prev = length(callee.tasks) + 1 @@ -4352,16 +4354,23 @@ function typeinf(interp::AbstractInterpreter, frame::InferenceState) reverse!(callee.tasks, prev) elseif callee.cycleid == length(callstack) # With no active ip's and no cycles, frame is done - finish_nocycle(interp, callee) + time_now = _time_ns() + callee.time_self_ns += (time_now - time_before) + time_before = time_now + finish_nocycle(interp, callee, time_before) callee.frameid == 0 && break takenext = length(callstack) nextstateid = takenext + 1 - frame.frameid #@assert length(nextstates) == nextstateid + 1 #@assert all(i -> !isdefined(nextstates[i], :result), nextstateid+1:length(nextstates)) resize!(nextstates, nextstateid) + continue elseif callee.cycleid == callee.frameid # If the current frame is the top part of a cycle, check if the whole cycle # is done, and if not, pick the next item to work on. + time_now = _time_ns() + callee.time_self_ns += (time_now - time_before) + time_before = time_now no_active_ips_in_cycle = true for i = callee.cycleid:length(callstack) caller = callstack[i]::InferenceState @@ -4372,7 +4381,7 @@ function typeinf(interp::AbstractInterpreter, frame::InferenceState) end end if no_active_ips_in_cycle - finish_cycle(interp, callstack, callee.cycleid) + finish_cycle(interp, callstack, callee.cycleid, time_before) end takenext = length(callstack) nextstateid = takenext + 1 - frame.frameid @@ -4382,10 +4391,14 @@ function typeinf(interp::AbstractInterpreter, frame::InferenceState) else #@assert length(nextstates) == nextstateid end + continue else # Continue to the next frame in this cycle takenext = takenext - 1 end + time_now = _time_ns() + callee.time_self_ns += (time_now - time_before) + time_before = time_now end #@assert all(nextresult -> !isdefined(nextresult, :result), nextstates) return is_inferred(frame) diff --git a/Compiler/src/inferencestate.jl b/Compiler/src/inferencestate.jl index 6fe52ea3e6f84..659b867128177 100644 --- a/Compiler/src/inferencestate.jl +++ b/Compiler/src/inferencestate.jl @@ -302,6 +302,10 @@ mutable struct InferenceState bestguess #::Type exc_bestguess ipo_effects::Effects + time_start::UInt64 + time_caches::Float64 + time_paused::UInt64 + time_self_ns::UInt64 #= flags =# # Whether to restrict inference of abstract call sites to avoid excessive work @@ -392,6 +396,7 @@ mutable struct InferenceState currbb, currpc, ip, handler_info, ssavalue_uses, bb_vartables, bb_saw_latestworld, ssavaluetypes, ssaflags, edges, stmt_info, tasks, pclimitations, limitations, cycle_backedges, callstack, parentid, frameid, cycleid, result, unreachable, bestguess, exc_bestguess, ipo_effects, + _time_ns(), 0.0, 0, 0, restrict_abstract_call_sites, cache_mode, insert_coverage, interp) @@ -815,6 +820,8 @@ mutable struct IRInterpretationState const mi::MethodInstance world::WorldWithRange curridx::Int + time_caches::Float64 + time_paused::UInt64 const argtypes_refined::Vector{Bool} const sptypes::Vector{VarState} const tpdum::TwoPhaseDefUseMap @@ -849,7 +856,8 @@ mutable struct IRInterpretationState tasks = WorkThunk[] edges = Any[] callstack = AbsIntState[] - return new(spec_info, ir, mi, WorldWithRange(world, valid_worlds), curridx, argtypes_refined, ir.sptypes, tpdum, + return new(spec_info, ir, mi, WorldWithRange(world, valid_worlds), + curridx, 0.0, 0, argtypes_refined, ir.sptypes, tpdum, ssa_refined, lazyreachability, tasks, edges, callstack, 0, 0) end end diff --git a/Compiler/src/typeinfer.jl b/Compiler/src/typeinfer.jl index 385cf0ef10894..e07ff4a842e3c 100644 --- a/Compiler/src/typeinfer.jl +++ b/Compiler/src/typeinfer.jl @@ -12,7 +12,7 @@ module Timings using ..Core using ..Compiler: -, +, :, Vector, length, first, empty!, push!, pop!, @inline, - @inbounds, copy, backtrace + @inbounds, copy, backtrace, _time_ns # What we record for any given frame we infer during type inference. struct InferenceFrameInfo @@ -53,8 +53,6 @@ end Timing(mi_info, start_time, cur_start_time, time, children) = Timing(mi_info, start_time, cur_start_time, time, children, nothing) Timing(mi_info, start_time) = Timing(mi_info, start_time, start_time, UInt64(0), Timing[]) -_time_ns() = ccall(:jl_hrtime, UInt64, ()) - # We keep a stack of the Timings for each of the MethodInstances currently being timed. # Since type inference currently operates via a depth-first search (during abstract # evaluation), this vector operates like a call stack. The last node in _timings is the @@ -93,7 +91,7 @@ If set to `true`, record per-method-instance timings within type inference in th __set_measure_typeinf(onoff::Bool) = __measure_typeinf__[] = onoff const __measure_typeinf__ = RefValue{Bool}(false) -function finish!(interp::AbstractInterpreter, caller::InferenceState, validation_world::UInt) +function finish!(interp::AbstractInterpreter, caller::InferenceState, validation_world::UInt, time_before::UInt64) result = caller.result opt = result.src if opt isa OptimizationState @@ -139,9 +137,12 @@ function finish!(interp::AbstractInterpreter, caller::InferenceState, validation if !@isdefined di di = DebugInfo(result.linfo) end - ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any), + time_now = _time_ns() + time_self_ns = caller.time_self_ns + (time_now - time_before) + time_total = (time_now - caller.time_start - caller.time_paused) * 1e-9 + ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), ci, inferred_result, const_flag, first(result.valid_worlds), last(result.valid_worlds), encode_effects(result.ipo_effects), - result.analysis_results, di, edges) + result.analysis_results, time_total, caller.time_caches, time_self_ns * 1e-9, di, edges) engine_reject(interp, ci) if !discard_src && isdefined(interp, :codegen) && uncompressed isa CodeInfo # record that the caller could use this result to generate code when required, if desired, to avoid repeating n^2 work @@ -182,8 +183,8 @@ function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstan end ccall(:jl_fill_codeinst, Cvoid, (Any, Any, Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any), ci, rettype, exctype, nothing, const_flags, min_world, max_world, ipo_effects, nothing, di, edges) - ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Any, Any), - ci, nothing, const_flag, min_world, max_world, ipo_effects, nothing, di, edges) + ccall(:jl_update_codeinst, Cvoid, (Any, Any, Int32, UInt, UInt, UInt32, Any, Float64, Float64, Float64, Any, Any), + ci, nothing, const_flag, min_world, max_world, ipo_effects, nothing, 0.0, 0.0, 0.0, di, edges) code_cache(interp)[mi] = ci if isdefined(interp, :codegen) interp.codegen[ci] = src @@ -192,14 +193,14 @@ function finish!(interp::AbstractInterpreter, mi::MethodInstance, ci::CodeInstan return nothing end -function finish_nocycle(::AbstractInterpreter, frame::InferenceState) +function finish_nocycle(::AbstractInterpreter, frame::InferenceState, time_before::UInt64) finishinfer!(frame, frame.interp, frame.cycleid) opt = frame.result.src if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` optimize(frame.interp, opt, frame.result) end validation_world = get_world_counter() - finish!(frame.interp, frame, validation_world) + finish!(frame.interp, frame, validation_world, time_before) if isdefined(frame.result, :ci) # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining @@ -214,7 +215,7 @@ function finish_nocycle(::AbstractInterpreter, frame::InferenceState) return nothing end -function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cycleid::Int) +function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cycleid::Int, time_before::UInt64) cycle_valid_worlds = WorldRange() cycle_valid_effects = EFFECTS_TOTAL for frameid = cycleid:length(frames) @@ -231,23 +232,45 @@ function finish_cycle(::AbstractInterpreter, frames::Vector{AbsIntState}, cyclei caller = frames[frameid]::InferenceState adjust_cycle_frame!(caller, cycle_valid_worlds, cycle_valid_effects) finishinfer!(caller, caller.interp, cycleid) + time_now = _time_ns() + caller.time_self_ns += (time_now - time_before) + time_before = time_now end + time_caches = 0.0 # the total and adjusted time of every entry in the cycle are the same + time_paused = UInt64(0) for frameid = cycleid:length(frames) caller = frames[frameid]::InferenceState opt = caller.result.src if opt isa OptimizationState # implies `may_optimize(caller.interp) === true` optimize(caller.interp, opt, caller.result) + time_now = _time_ns() + caller.time_self_ns += (time_now - time_before) + time_before = time_now end + time_caches += caller.time_caches + time_paused += caller.time_paused + caller.time_paused = UInt64(0) + caller.time_caches = 0.0 end + cycletop = frames[cycleid]::InferenceState + time_start = cycletop.time_start validation_world = get_world_counter() cis = CodeInstance[] for frameid = cycleid:length(frames) caller = frames[frameid]::InferenceState - finish!(caller.interp, caller, validation_world) + caller.time_start = time_start + caller.time_caches = time_caches + caller.time_paused = time_paused + finish!(caller.interp, caller, validation_world, time_before) if isdefined(caller.result, :ci) push!(cis, caller.result.ci) end end + if cycletop.parentid != 0 + parent = frames[cycletop.parentid] + parent.time_caches += time_caches + parent.time_paused += time_paused + end # After validation, under the world_counter_lock, set max_world to typemax(UInt) for all dependencies # (recursively). From that point onward the ordinary backedge mechanism is responsible for maintaining # validity. @@ -778,9 +801,10 @@ function return_cached_result(interp::AbstractInterpreter, method::Method, codei rt = cached_return_type(codeinst) exct = codeinst.exctype effects = ipo_effects(codeinst) - edge = codeinst update_valid_age!(caller, WorldRange(min_world(codeinst), max_world(codeinst))) - return Future(MethodCallResult(interp, caller, method, rt, exct, effects, edge, edgecycle, edgelimited)) + caller.time_caches += reinterpret(Float16, codeinst.time_infer_total) + caller.time_caches += reinterpret(Float16, codeinst.time_infer_cache_saved) + return Future(MethodCallResult(interp, caller, method, rt, exct, effects, codeinst, edgecycle, edgelimited)) end function MethodCallResult(::AbstractInterpreter, sv::AbsIntState, method::Method, @@ -876,7 +900,9 @@ function typeinf_edge(interp::AbstractInterpreter, method::Method, @nospecialize if frame === false # completely new, but check again after reserving in the engine if cache_mode == CACHE_MODE_GLOBAL + reserve_start = _time_ns() # subtract engine_reserve (thread-synchronization) time from callers to avoid double-counting ci_from_engine = engine_reserve(interp, mi) + caller.time_paused += (_time_ns() - reserve_start) edge_ci = ci_from_engine codeinst = get(code_cache(interp), mi, nothing) if codeinst isa CodeInstance # return existing rettype if the code is already inferred diff --git a/Compiler/src/utilities.jl b/Compiler/src/utilities.jl index 6113bdb14e7df..477c518518918 100644 --- a/Compiler/src/utilities.jl +++ b/Compiler/src/utilities.jl @@ -351,3 +351,5 @@ function inbounds_option() end is_asserts() = ccall(:jl_is_assertsbuild, Cint, ()) == 1 + +_time_ns() = ccall(:jl_hrtime, UInt64, ()) diff --git a/base/Base.jl b/base/Base.jl index 7b1672bf9602c..c6e33df751782 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -36,9 +36,7 @@ include("views.jl") # numeric operations include("hashing.jl") -include("rounding.jl") include("div.jl") -include("float.jl") include("twiceprecision.jl") include("complex.jl") include("rational.jl") diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index 2bd9a75a6dca6..640b8226788cb 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -277,6 +277,8 @@ include("operators.jl") include("pointer.jl") include("refvalue.jl") include("cmem.jl") +include("rounding.jl") +include("float.jl") include("checked.jl") using .Checked diff --git a/base/essentials.jl b/base/essentials.jl index 1e2ec0ef79103..93c36968adb7c 100644 --- a/base/essentials.jl +++ b/base/essentials.jl @@ -323,7 +323,7 @@ macro _nothrow_meta() #=:consistent_overlay=#false, #=:nortcall=#false)) end -# can be used in place of `@assume_effects :nothrow` (supposed to be used for bootstrapping) +# can be used in place of `@assume_effects :noub` (supposed to be used for bootstrapping) macro _noub_meta() return _is_internal(__module__) && Expr(:meta, Expr(:purity, #=:consistent=#false, diff --git a/base/float.jl b/base/float.jl index 54e232a01b8cb..1878a6a953360 100644 --- a/base/float.jl +++ b/base/float.jl @@ -530,7 +530,8 @@ function _to_float(number::U, ep) where {U<:Unsigned} return reinterpret(F, bits) end -@assume_effects :terminates_locally :nothrow function rem_internal(x::T, y::T) where {T<:IEEEFloat} +function rem_internal(x::T, y::T) where {T<:IEEEFloat} + @_terminates_locally_meta xuint = reinterpret(Unsigned, x) yuint = reinterpret(Unsigned, y) if xuint <= yuint @@ -625,13 +626,15 @@ end isequal(x::T, y::T) where {T<:IEEEFloat} = fpiseq(x, y) # interpret as sign-magnitude integer -@inline function _fpint(x) +function _fpint(x) + @inline IntT = inttype(typeof(x)) ix = reinterpret(IntT, x) return ifelse(ix < zero(IntT), ix ⊻ typemax(IntT), ix) end -@inline function isless(a::T, b::T) where T<:IEEEFloat +function isless(a::T, b::T) where T<:IEEEFloat + @inline (isnan(a) || isnan(b)) && return !isnan(a) return _fpint(a) < _fpint(b) @@ -720,84 +723,6 @@ See also: [`Inf`](@ref), [`iszero`](@ref), [`isfinite`](@ref), [`isnan`](@ref). isinf(x::Real) = !isnan(x) & !isfinite(x) isinf(x::IEEEFloat) = abs(x) === oftype(x, Inf) -const hx_NaN = hash_uint64(reinterpret(UInt64, NaN)) -function hash(x::Float64, h::UInt) - # see comments on trunc and hash(Real, UInt) - if typemin(Int64) <= x < typemax(Int64) - xi = fptosi(Int64, x) - if isequal(xi, x) - return hash(xi, h) - end - elseif typemin(UInt64) <= x < typemax(UInt64) - xu = fptoui(UInt64, x) - if isequal(xu, x) - return hash(xu, h) - end - elseif isnan(x) - return hx_NaN ⊻ h # NaN does not have a stable bit pattern - end - return hash_uint64(bitcast(UInt64, x)) - 3h -end - -hash(x::Float32, h::UInt) = hash(Float64(x), h) - -function hash(x::Float16, h::UInt) - # see comments on trunc and hash(Real, UInt) - if isfinite(x) # all finite Float16 fit in Int64 - xi = fptosi(Int64, x) - if isequal(xi, x) - return hash(xi, h) - end - elseif isnan(x) - return hx_NaN ⊻ h # NaN does not have a stable bit pattern - end - return hash_uint64(bitcast(UInt64, Float64(x))) - 3h -end - -## generic hashing for rational values ## -function hash(x::Real, h::UInt) - # decompose x as num*2^pow/den - num, pow, den = decompose(x) - - # handle special values - num == 0 && den == 0 && return hash(NaN, h) - num == 0 && return hash(ifelse(den > 0, 0.0, -0.0), h) - den == 0 && return hash(ifelse(num > 0, Inf, -Inf), h) - - # normalize decomposition - if den < 0 - num = -num - den = -den - end - num_z = trailing_zeros(num) - num >>= num_z - den_z = trailing_zeros(den) - den >>= den_z - pow += num_z - den_z - # If the real can be represented as an Int64, UInt64, or Float64, hash as those types. - # To be an Integer the denominator must be 1 and the power must be non-negative. - if den == 1 - # left = ceil(log2(num*2^pow)) - left = top_set_bit(abs(num)) + pow - # 2^-1074 is the minimum Float64 so if the power is smaller, not a Float64 - if -1074 <= pow - if 0 <= pow # if pow is non-negative, it is an integer - left <= 63 && return hash(Int64(num) << Int(pow), h) - left <= 64 && !signbit(num) && return hash(UInt64(num) << Int(pow), h) - end # typemin(Int64) handled by Float64 case - # 2^1024 is the maximum Float64 so if the power is greater, not a Float64 - # Float64s only have 53 mantisa bits (including implicit bit) - left <= 1024 && left - pow <= 53 && return hash(ldexp(Float64(num), pow), h) - end - else - h = hash_integer(den, h) - end - # handle generic rational values - h = hash_integer(pow, h) - h = hash_integer(num, h) - return h -end - #= `decompose(x)`: non-canonical decomposition of rational values as `num*2^pow/den`. diff --git a/base/hashing.jl b/base/hashing.jl index d4a6217de6edb..868f5e1ad9abf 100644 --- a/base/hashing.jl +++ b/base/hashing.jl @@ -98,6 +98,87 @@ function hash_integer(n::Integer, h::UInt) return h end +## efficient value-based hashing of floats ## + +const hx_NaN = hash_uint64(reinterpret(UInt64, NaN)) +function hash(x::Float64, h::UInt) + # see comments on trunc and hash(Real, UInt) + if typemin(Int64) <= x < typemax(Int64) + xi = fptosi(Int64, x) + if isequal(xi, x) + return hash(xi, h) + end + elseif typemin(UInt64) <= x < typemax(UInt64) + xu = fptoui(UInt64, x) + if isequal(xu, x) + return hash(xu, h) + end + elseif isnan(x) + return hx_NaN ⊻ h # NaN does not have a stable bit pattern + end + return hash_uint64(bitcast(UInt64, x)) - 3h +end + +hash(x::Float32, h::UInt) = hash(Float64(x), h) + +function hash(x::Float16, h::UInt) + # see comments on trunc and hash(Real, UInt) + if isfinite(x) # all finite Float16 fit in Int64 + xi = fptosi(Int64, x) + if isequal(xi, x) + return hash(xi, h) + end + elseif isnan(x) + return hx_NaN ⊻ h # NaN does not have a stable bit pattern + end + return hash_uint64(bitcast(UInt64, Float64(x))) - 3h +end + +## generic hashing for rational values ## +function hash(x::Real, h::UInt) + # decompose x as num*2^pow/den + num, pow, den = decompose(x) + + # handle special values + num == 0 && den == 0 && return hash(NaN, h) + num == 0 && return hash(ifelse(den > 0, 0.0, -0.0), h) + den == 0 && return hash(ifelse(num > 0, Inf, -Inf), h) + + # normalize decomposition + if den < 0 + num = -num + den = -den + end + num_z = trailing_zeros(num) + num >>= num_z + den_z = trailing_zeros(den) + den >>= den_z + pow += num_z - den_z + # If the real can be represented as an Int64, UInt64, or Float64, hash as those types. + # To be an Integer the denominator must be 1 and the power must be non-negative. + if den == 1 + # left = ceil(log2(num*2^pow)) + left = top_set_bit(abs(num)) + pow + # 2^-1074 is the minimum Float64 so if the power is smaller, not a Float64 + if -1074 <= pow + if 0 <= pow # if pow is non-negative, it is an integer + left <= 63 && return hash(Int64(num) << Int(pow), h) + left <= 64 && !signbit(num) && return hash(UInt64(num) << Int(pow), h) + end # typemin(Int64) handled by Float64 case + # 2^1024 is the maximum Float64 so if the power is greater, not a Float64 + # Float64s only have 53 mantisa bits (including implicit bit) + left <= 1024 && left - pow <= 53 && return hash(ldexp(Float64(num), pow), h) + end + else + h = hash_integer(den, h) + end + # handle generic rational values + h = hash_integer(pow, h) + h = hash_integer(num, h) + return h +end + + ## symbol & expression hashing ## if UInt === UInt64 diff --git a/base/rounding.jl b/base/rounding.jl index 98b4c30822245..5865c9aef3b5f 100644 --- a/base/rounding.jl +++ b/base/rounding.jl @@ -2,7 +2,7 @@ module Rounding -let fenv_consts = Vector{Cint}(undef, 9) +let fenv_consts = Array{Cint,1}(undef, 9) ccall(:jl_get_fenv_consts, Cvoid, (Ptr{Cint},), fenv_consts) global const JL_FE_INEXACT = fenv_consts[1] global const JL_FE_UNDERFLOW = fenv_consts[2] diff --git a/doc/src/devdocs/ast.md b/doc/src/devdocs/ast.md index 706dfc34875fa..1330940862b99 100644 --- a/doc/src/devdocs/ast.md +++ b/doc/src/devdocs/ast.md @@ -661,6 +661,26 @@ for important details on how to modify these fields safely. If max_world is the special token value `-1`, the value is not yet known. It may continue to be used until we encounter a backedge that requires us to reconsider. + * Timing fields + + - `time_infer_total`: Total cost of computing `inferred` originally as wall-time from start to finish. + + - `time_infer_cache_saved`: The cost saved from `time_infer_total` by having caching. + Adding this to `time_infer_total` should give a stable estimate for comparing the cost + of two implementations or one implementation over time. This is generally an + over-estimate of the time to infer something, since the cache is frequently effective + at handling repeated work. + + - `time_infer_self`: Self cost of julia inference for `inferred` (a portion of + `time_infer_total`). This is simply the incremental cost of compiling this one method, + if given a fully populated cache of all call targets, even including constant + inference results and LimitedAccuracy results, which generally are not in a cache. + + - `time_compile`: Self cost of llvm JIT compilation (e.g. of computing `invoke` from + `inferred`). A total cost estimate can be computed by walking all of the `edges` + contents and summing those, while accounting for cycles and duplicates. (This field + currently does not include any measured AOT compile times.) + ### CodeInfo diff --git a/src/gf.c b/src/gf.c index 813f9dfaca328..cc3966da5f393 100644 --- a/src/gf.c +++ b/src/gf.c @@ -640,6 +640,9 @@ JL_DLLEXPORT jl_code_instance_t *jl_new_codeinst( assert(const_flags & 2); jl_atomic_store_relaxed(&codeinst->invoke, jl_fptr_const_return); } + codeinst->time_infer_total = 0; + codeinst->time_infer_self = 0; + jl_atomic_store_relaxed(&codeinst->time_compile, 0); jl_atomic_store_relaxed(&codeinst->specsigflags, 0); jl_atomic_store_relaxed(&codeinst->precompile, 0); jl_atomic_store_relaxed(&codeinst->next, NULL); @@ -652,12 +655,16 @@ JL_DLLEXPORT void jl_update_codeinst( jl_code_instance_t *codeinst, jl_value_t *inferred, int32_t const_flags, size_t min_world, size_t max_world, uint32_t effects, jl_value_t *analysis_results, + double time_infer_total, double time_infer_cache_saved, double time_infer_self, jl_debuginfo_t *di, jl_svec_t *edges /* , int absolute_max*/) { assert(min_world <= max_world && "attempting to set invalid world constraints"); //assert((!jl_is_method(codeinst->def->def.value) || max_world != ~(size_t)0 || min_world <= 1 || jl_svec_len(edges) != 0) && "missing edges"); codeinst->analysis_results = analysis_results; jl_gc_wb(codeinst, analysis_results); + codeinst->time_infer_total = julia_double_to_half(time_infer_total); + codeinst->time_infer_cache_saved = julia_double_to_half(time_infer_cache_saved); + codeinst->time_infer_self = julia_double_to_half(time_infer_self); jl_atomic_store_relaxed(&codeinst->ipo_purity_bits, effects); jl_atomic_store_relaxed(&codeinst->debuginfo, di); jl_gc_wb(codeinst, di); diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 80642bef95619..695953b602653 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -627,15 +627,18 @@ static void jl_compile_codeinst_now(jl_code_instance_t *codeinst) // If logging of the compilation stream is enabled, // then dump the method-instance specialization type to the stream jl_method_instance_t *mi = jl_get_ci_mi(codeinst); + uint64_t end_time = jl_hrtime(); if (jl_is_method(mi->def.method)) { auto stream = *jl_ExecutionEngine->get_dump_compiles_stream(); if (stream) { - uint64_t end_time = jl_hrtime(); ios_printf(stream, "%" PRIu64 "\t\"", end_time - start_time); jl_static_show((JL_STREAM*)stream, mi->specTypes); ios_printf(stream, "\"\n"); } } + jl_atomic_store_relaxed(&codeinst->time_compile, + julia_double_to_half(julia_half_to_float(jl_atomic_load_relaxed(&codeinst->time_compile)) + + (end_time - start_time) * 1e-9)); lock.native.lock(); } else { diff --git a/src/jltypes.c b/src/jltypes.c index d2dbfbc4fe439..151fa0ce46d40 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3643,7 +3643,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_code_instance_type = jl_new_datatype(jl_symbol("CodeInstance"), core, jl_any_type, jl_emptysvec, - jl_perm_symsvec(17, + jl_perm_symsvec(21, "def", "owner", "next", @@ -3655,12 +3655,16 @@ void jl_init_types(void) JL_GC_DISABLED "inferred", "debuginfo", "edges", - //"absolute_max", - "ipo_purity_bits", "analysis_results", + "ipo_purity_bits", + "time_infer_total", + "time_infer_cache_saved", + "time_infer_self", + "time_compile", + //"absolute_max", "specsigflags", "precompile", "invoke", "specptr"), // function object decls - jl_svec(17, + jl_svec(21, jl_any_type, jl_any_type, jl_any_type, @@ -3672,17 +3676,21 @@ void jl_init_types(void) JL_GC_DISABLED jl_any_type, jl_debuginfo_type, jl_simplevector_type, - //jl_bool_type, - jl_uint32_type, jl_any_type, + jl_uint32_type, + jl_uint16_type, + jl_uint16_type, + jl_uint16_type, + jl_uint16_type, + //jl_bool_type, jl_bool_type, jl_bool_type, jl_any_type, jl_any_type), // fptrs jl_emptysvec, 0, 1, 1); jl_svecset(jl_code_instance_type->types, 2, jl_code_instance_type); - const static uint32_t code_instance_constfields[1] = { 0b00001000011100011 }; // Set fields 1, 2, 6-8, 13 as const - const static uint32_t code_instance_atomicfields[1] = { 0b11110111100011100 }; // Set fields 3-5, 9-12, 14-17 as atomic + const static uint32_t code_instance_constfields[1] = { 0b000001110100011100011 }; // Set fields 1, 2, 6-8, 12, 14-16 as const + const static uint32_t code_instance_atomicfields[1] = { 0b111110001011100011100 }; // Set fields 3-5, 9-12, 13, 17-21 as atomic // Fields 4-5 are only operated on by construction and deserialization, so are effectively const at runtime // Fields ipo_purity_bits and analysis_results are not currently threadsafe or reliable, as they get mutated after optimization, but are not declared atomic // and there is no way to tell (during inference) if their value is finalized yet (to wait for them to be narrowed if applicable) @@ -3858,8 +3866,8 @@ void jl_init_types(void) JL_GC_DISABLED jl_svecset(jl_method_type->types, 13, jl_method_instance_type); //jl_svecset(jl_debuginfo_type->types, 0, jl_method_instance_type); // union(jl_method_instance_type, jl_method_type, jl_symbol_type) jl_svecset(jl_method_instance_type->types, 4, jl_code_instance_type); - jl_svecset(jl_code_instance_type->types, 15, jl_voidpointer_type); - jl_svecset(jl_code_instance_type->types, 16, jl_voidpointer_type); + jl_svecset(jl_code_instance_type->types, 19, jl_voidpointer_type); + jl_svecset(jl_code_instance_type->types, 20, jl_voidpointer_type); jl_svecset(jl_binding_type->types, 0, jl_globalref_type); jl_svecset(jl_binding_type->types, 3, jl_array_any_type); jl_svecset(jl_binding_partition_type->types, 3, jl_binding_partition_type); diff --git a/src/julia.h b/src/julia.h index 306f6a914eaac..694e16ca31a97 100644 --- a/src/julia.h +++ b/src/julia.h @@ -450,9 +450,9 @@ typedef struct _jl_code_instance_t { _Atomic(jl_value_t *) inferred; _Atomic(jl_debuginfo_t *) debuginfo; // stored information about edges from this object (set once, with a happens-before both source and invoke) _Atomic(jl_svec_t *) edges; // forward edge info - //TODO: uint8_t absolute_max; // whether true max world is unknown // purity results + jl_value_t *analysis_results; // Analysis results about this code (IPO-safe) // see also encode_effects() and decode_effects() in `base/compiler/effects.jl`, _Atomic(uint32_t) ipo_purity_bits; // purity_flags: @@ -464,9 +464,14 @@ typedef struct _jl_code_instance_t { // uint8_t inaccessiblememonly : 2; // uint8_t noub : 2; // uint8_t nonoverlayed : 2; - jl_value_t *analysis_results; // Analysis results about this code (IPO-safe) // compilation state cache + // these time fields have units of seconds (60 ns minimum resolution and 18 hour maximum saturates to Infinity) and are stored in Float16 format + uint16_t time_infer_total; // total cost of computing `inferred` originally + uint16_t time_infer_cache_saved; // adjustment to total cost, reflecting how much time was saved by having caches, to give a stable real cost without caches for comparisons + uint16_t time_infer_self; // self cost of julia inference for `inferred` (included in time_infer_total) + _Atomic(uint16_t) time_compile; // self cost of llvm compilation (e.g. of computing `invoke`) + //TODO: uint8_t absolute_max; // whether true max world is unknown _Atomic(uint8_t) specsigflags; // & 0b001 == specptr is a specialized function signature for specTypes->rettype // & 0b010 == invokeptr matches specptr // & 0b100 == From image diff --git a/src/julia_internal.h b/src/julia_internal.h index 5f29e9462b816..8d02cce9a80d9 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -1708,6 +1708,10 @@ JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary); JL_DLLEXPORT uintptr_t jl_object_id_(uintptr_t tv, jl_value_t *v) JL_NOTSAFEPOINT; JL_DLLEXPORT void jl_set_next_task(jl_task_t *task) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint16_t julia_double_to_half(double param) JL_NOTSAFEPOINT; +JL_DLLEXPORT uint16_t julia_float_to_half(float param) JL_NOTSAFEPOINT; +JL_DLLEXPORT float julia_half_to_float(uint16_t param) JL_NOTSAFEPOINT; + // -- synchronization utilities -- // extern jl_mutex_t typecache_lock; @@ -1989,13 +1993,6 @@ jl_sym_t *_jl_symbol(const char *str, size_t len) JL_NOTSAFEPOINT; #define JL_WEAK_SYMBOL_DEFAULT(sym) NULL #endif -//JL_DLLEXPORT float julia__gnu_h2f_ieee(half param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT half julia__gnu_f2h_ieee(float param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT half julia__truncdfhf2(double param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT float julia__truncsfbf2(float param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT float julia__truncdfbf2(double param) JL_NOTSAFEPOINT; -//JL_DLLEXPORT double julia__extendhfdf2(half n) JL_NOTSAFEPOINT; - JL_DLLEXPORT uint32_t jl_crc32c(uint32_t crc, const char *buf, size_t len); // -- exports from codegen -- // diff --git a/src/staticdata.c b/src/staticdata.c index ec6d706bace9f..4671b5c1fecf6 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -1881,6 +1881,7 @@ static void jl_write_values(jl_serializer_state *s) JL_GC_DISABLED jl_atomic_store_release(&newci->max_world, 0); } } + jl_atomic_store_relaxed(&newci->time_compile, 0.0); jl_atomic_store_relaxed(&newci->invoke, NULL); jl_atomic_store_relaxed(&newci->specsigflags, 0); jl_atomic_store_relaxed(&newci->specptr.fptr, NULL); diff --git a/test/core.jl b/test/core.jl index 036ca63eb70b4..8895d3a7bb86e 100644 --- a/test/core.jl +++ b/test/core.jl @@ -14,7 +14,7 @@ include("testenv.jl") # sanity tests that our built-in types are marked correctly for const fields for (T, c) in ( (Core.CodeInfo, []), - (Core.CodeInstance, [:def, :owner, :rettype, :exctype, :rettype_const, :analysis_results]), + (Core.CodeInstance, [:def, :owner, :rettype, :exctype, :rettype_const, :analysis_results, :time_infer_total, :time_infer_cache_saved, :time_infer_self]), (Core.Method, [#=:name, :module, :file, :line, :primary_world, :sig, :slot_syms, :external_mt, :nargs, :called, :nospecialize, :nkw, :isva, :is_for_opaque_closure, :constprop=#]), (Core.MethodInstance, [#=:def, :specTypes, :sparam_vals=#]), (Core.MethodTable, [:module]), @@ -33,7 +33,7 @@ end # sanity tests that our built-in types are marked correctly for atomic fields for (T, c) in ( (Core.CodeInfo, []), - (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :edges, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile]), + (Core.CodeInstance, [:next, :min_world, :max_world, :inferred, :edges, :debuginfo, :ipo_purity_bits, :invoke, :specptr, :specsigflags, :precompile, :time_compile]), (Core.Method, [:primary_world, :deleted_world]), (Core.MethodInstance, [:cache, :flags]), (Core.MethodTable, [:defs, :leafcache, :cache, :max_args]), From 210288bb787fbb44858662190741b2f1beb0cb18 Mon Sep 17 00:00:00 2001 From: Gabriel Baraldi Date: Thu, 20 Mar 2025 11:16:55 -0300 Subject: [PATCH 66/74] Mark UInt8 field as UInt8 in CodInstance (#57822) Otherwise inspecting it from julia will be wrong/UB (cherry picked from commit 0df5ad78d0f010edd92b15663a9d2c7fc67bc2b3) --- src/jltypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jltypes.c b/src/jltypes.c index 151fa0ce46d40..dd8482cfea70a 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -3683,7 +3683,7 @@ void jl_init_types(void) JL_GC_DISABLED jl_uint16_type, jl_uint16_type, //jl_bool_type, - jl_bool_type, + jl_uint8_type, jl_bool_type, jl_any_type, jl_any_type), // fptrs jl_emptysvec, From 8cb30fa250f18fb6859e42f2513fa53d14731bf4 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Mon, 10 Mar 2025 22:03:28 -0400 Subject: [PATCH 67/74] trimming: Restore JLL support (#57587) Add an extra indirection to our printing code for `@assert` so that that inferrability is restored after the override. Also partially revert d77c24f009b93577bcf254908e639c9f149f2e4e which, deleted init code important for JLL's ~This still requires that we bypass this inference-disabling code in JLLWrappers.jl:~ ```julia if isdefined(Base, :Experimental) && isdefined(Base.Experimental, Symbol("@compiler_options")) @eval Base.Experimental.@compiler_options compile=min optimize=0 infer=false end ``` (cherry picked from commit d369c10c6fe815e34621eda4c5bc5fbc6e6feb36) --- base/error.jl | 5 +++-- base/experimental.jl | 2 +- contrib/juliac-buildscript.jl | 3 --- test/trimming/Makefile | 13 ++++++++++--- test/trimming/Project.toml | 3 +++ test/trimming/basic_jll.jl | 14 ++++++++++++++ test/trimming/trimming.jl | 14 +++++++++++--- 7 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 test/trimming/Project.toml create mode 100644 test/trimming/basic_jll.jl diff --git a/base/error.jl b/base/error.jl index 276555033443a..3ea7210652dad 100644 --- a/base/error.jl +++ b/base/error.jl @@ -240,13 +240,14 @@ macro assert(ex, msgs...) msg = Main.Base.string(msg) else # string() might not be defined during bootstrap - msg = :(Main.Base.inferencebarrier(_assert_tostring)($(Expr(:quote,msg)))) + msg = :(_assert_tostring($(Expr(:quote,msg)))) end return :($(esc(ex)) ? $(nothing) : throw(AssertionError($msg))) end # this may be overridden in contexts where `string(::Expr)` doesn't work -_assert_tostring(msg) = isdefined(Main, :Base) ? Main.Base.string(msg) : +_assert_tostring(@nospecialize(msg)) = Core.compilerbarrier(:type, __assert_tostring)(msg) +__assert_tostring(msg) = isdefined(Main, :Base) ? Main.Base.string(msg) : (Core.println(msg); "Error during bootstrap. See stdout.") struct ExponentialBackOff diff --git a/base/experimental.jl b/base/experimental.jl index f56f8d8234282..efaf4bd33a820 100644 --- a/base/experimental.jl +++ b/base/experimental.jl @@ -163,7 +163,7 @@ macro max_methods(n::Int, fdef::Expr) end """ - Experimental.@compiler_options optimize={0,1,2,3} compile={yes,no,all,min} infer={yes,no} max_methods={default,1,2,3,4} + Experimental.@compiler_options optimize={0,1,2,3} compile={yes,no,all,min} infer={true,false} max_methods={default,1,2,3,4} Set compiler options for code in the enclosing module. Options correspond directly to command-line options with the same name, where applicable. The following options diff --git a/contrib/juliac-buildscript.jl b/contrib/juliac-buildscript.jl index f51238cc0862a..363330e4a3a79 100644 --- a/contrib/juliac-buildscript.jl +++ b/contrib/juliac-buildscript.jl @@ -36,9 +36,6 @@ end JuliaSyntax.enable_in_core!() = nothing init_active_project() = ACTIVE_PROJECT[] = nothing set_active_project(projfile::Union{AbstractString,Nothing}) = ACTIVE_PROJECT[] = projfile - init_depot_path() = nothing - init_load_path() = nothing - init_active_project() = nothing disable_library_threading() = nothing start_profile_listener() = nothing invokelatest_trimmed(f, args...; kwargs...) = f(args...; kwargs...) diff --git a/test/trimming/Makefile b/test/trimming/Makefile index d2da21eb71a88..bb2b64c2d0dd5 100644 --- a/test/trimming/Makefile +++ b/test/trimming/Makefile @@ -30,7 +30,7 @@ LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) -ljulia-internal #============================================================================= -release: hello$(EXE) +release: hello$(EXE) basic_jll$(EXE) hello.o: $(SRCDIR)/hello.jl $(BUILDSCRIPT) $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.so --startup-file=no --history-file=no --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $(SRCDIR)/hello.jl --output-exe true @@ -38,14 +38,21 @@ hello.o: $(SRCDIR)/hello.jl $(BUILDSCRIPT) init.o: $(SRCDIR)/init.c $(CC) -c -o $@ $< $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) +basic_jll.o: $(SRCDIR)/basic_jll.jl $(BUILDSCRIPT) + $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.so --startup-file=no --history-file=no --project=$(SRCDIR) -e "using Pkg; Pkg.instantiate()" + $(JULIA) -t 1 -J $(BIN)/../lib/julia/sys.so --startup-file=no --history-file=no --project=$(SRCDIR) --output-o $@ --output-incremental=no --strip-ir --strip-metadata --experimental --trim $(BUILDSCRIPT) $(SRCDIR)/basic_jll.jl --output-exe true + hello$(EXE): hello.o init.o $(CC) -o $@ $(WHOLE_ARCHIVE) hello.o $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) -check: hello$(EXE) +basic_jll$(EXE): basic_jll.o init.o + $(CC) -o $@ $(WHOLE_ARCHIVE) basic_jll.o $(NO_WHOLE_ARCHIVE) init.o $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) + +check: hello$(EXE) basic_jll$(EXE) $(JULIA) --depwarn=error $(SRCDIR)/../runtests.jl $(SRCDIR)/trimming clean: - -rm -f hello$(EXE) init.o hello.o + -rm -f hello$(EXE) basic_jll$(EXE) init.o hello.o basic_jll.o .PHONY: release clean check diff --git a/test/trimming/Project.toml b/test/trimming/Project.toml new file mode 100644 index 0000000000000..a0bf6688d9dd4 --- /dev/null +++ b/test/trimming/Project.toml @@ -0,0 +1,3 @@ +[deps] +JLLWrappers = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +Zstd_jll = "3161d3a3-bdf6-5164-811a-617609db77b4" diff --git a/test/trimming/basic_jll.jl b/test/trimming/basic_jll.jl new file mode 100644 index 0000000000000..fc0137dd4eab2 --- /dev/null +++ b/test/trimming/basic_jll.jl @@ -0,0 +1,14 @@ +module MyApp + +using Libdl +using Zstd_jll + +Base.@ccallable function main()::Cint + println(Core.stdout, "Julia! Hello, world!") + fptr = dlsym(Zstd_jll.libzstd_handle, :ZSTD_versionString) + println(Core.stdout, unsafe_string(ccall(fptr, Cstring, ()))) + println(Core.stdout, unsafe_string(ccall((:ZSTD_versionString, libzstd), Cstring, ()))) + return 0 +end + +end diff --git a/test/trimming/trimming.jl b/test/trimming/trimming.jl index dfacae7f8e531..0c5226cba01fe 100644 --- a/test/trimming/trimming.jl +++ b/test/trimming/trimming.jl @@ -1,7 +1,15 @@ using Test -exe_path = joinpath(@__DIR__, "hello"*splitext(Base.julia_exename())[2]) +let exe_suffix = splitext(Base.julia_exename())[2] -@test readchomp(`$exe_path`) == "Hello, world!" + hello_exe = joinpath(@__DIR__, "hello" * exe_suffix) + @test readchomp(`$hello_exe`) == "Hello, world!" + @test filesize(hello_exe) < filesize(unsafe_string(Base.JLOptions().image_file))/10 -@test filesize(exe_path) < filesize(unsafe_string(Base.JLOptions().image_file))/10 + basic_jll_exe = joinpath(@__DIR__, "basic_jll" * exe_suffix) + lines = split(readchomp(`$basic_jll_exe`), "\n") + @test lines[1] == "Julia! Hello, world!" + @test lines[2] == lines[3] + @test Base.VersionNumber(lines[2]) ≥ v"1.5.7" + @test filesize(basic_jll_exe) < filesize(unsafe_string(Base.JLOptions().image_file))/10 +end From f851c82b168b3553b60a3369a94a5621643679e6 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 14 Mar 2025 14:56:27 -0400 Subject: [PATCH 68/74] better behaved loading errors (#57727) 3 interrelated fixes here: - Re-word the confusing error message for a circular dependency error. - Calling `using` inside a package was previously declared to be an error, but never really enforced, so allow it explicitly to do what it logically means (when appearing explicitly, such as from having `Requires.@require` in an `__init__` function). Remove the associated warning in the docs, since this gotcha now simply means what users think it would mean. - A missing lock acquire in create_expr_cache lead to a confusing discrepancy in errors related to the previous two fixes between incremental and serial modes. Fixes https://github.com/JuliaLang/julia/issues/56742 (cherry picked from commit 7f8ca291adb4dbc8f9f60227c52b6f7c3b990c03) --- Makefile | 1 + NEWS.md | 4 +++ base/loading.jl | 55 +++++++++++++++++++++++---------------- doc/src/manual/modules.md | 30 ++++++++++----------- test/loading.jl | 4 +-- test/precompile.jl | 42 +++++++++++++++++++++--------- 6 files changed, 84 insertions(+), 52 deletions(-) diff --git a/Makefile b/Makefile index a7562946c1699..3a529177b62de 100644 --- a/Makefile +++ b/Makefile @@ -177,6 +177,7 @@ release-candidate: release testall @echo 14. Push to Juliaup (https://github.com/JuliaLang/juliaup/wiki/Adding-a-Julia-version) @echo 15. Announce on mailing lists @echo 16. Change master to release-0.X in base/version.jl and base/version_git.sh as in 4cb1e20 + @echo 17. Move NEWS.md contents to HISTORY.md @echo $(build_man1dir)/julia.1: $(JULIAHOME)/doc/man/julia.1 | $(build_man1dir) diff --git a/NEWS.md b/NEWS.md index 615b495018c18..c421b7eeb3f4e 100644 --- a/NEWS.md +++ b/NEWS.md @@ -52,6 +52,10 @@ Language changes * Errors during `getfield` now raise a new `FieldError` exception type instead of the generic `ErrorException` ([#54504]). * Macros in function-signature-position no longer require parentheses. E.g. `function @main(args) ... end` is now permitted, whereas `function (@main)(args) ... end` was required in prior Julia versions. +* Calling `using` on a package name inside of that package of that name (especially relevant + for a submodule) now explicitly uses that package without examining the Manifest and + environment, which is identical to the behavior of `..Name`. This appears to better match + how users expect this to behave in the wild. ([#57727]) Compiler/Runtime improvements diff --git a/base/loading.jl b/base/loading.jl index 818bd2366b2d0..89eecb3ec1fbc 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1438,7 +1438,6 @@ function run_module_init(mod::Module, i::Int=1) end function run_package_callbacks(modkey::PkgId) - @assert modkey != precompilation_target run_extension_callbacks(modkey) assert_havelock(require_lock) unlock(require_lock) @@ -1568,7 +1567,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any} uuid_trigger = UUID(totaldeps[trigger]::String) trigger_id = PkgId(uuid_trigger, trigger) push!(trigger_ids, trigger_id) - if !haskey(Base.loaded_modules, trigger_id) || haskey(package_locks, trigger_id) || (trigger_id == precompilation_target) + if !haskey(Base.loaded_modules, trigger_id) || haskey(package_locks, trigger_id) trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id) push!(trigger1, gid) else @@ -1581,7 +1580,6 @@ end loading_extension::Bool = false loadable_extensions::Union{Nothing,Vector{PkgId}} = nothing precompiling_extension::Bool = false -precompilation_target::Union{Nothing,PkgId} = nothing function run_extension_callbacks(extid::ExtensionId) assert_havelock(require_lock) succeeded = try @@ -2178,7 +2176,6 @@ function canstart_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool) # load already in progress for this module on the task task, cond = loading deps = String[modkey.name] - pkgid = modkey assert_havelock(cond.lock) if debug_loading_deadlocks && current_task() !== task waiters = Dict{Task,Pair{Task,PkgId}}() # invert to track waiting tasks => loading tasks @@ -2198,18 +2195,26 @@ function canstart_loading(modkey::PkgId, build_id::UInt128, stalecheck::Bool) end end if current_task() === task - others = String[modkey.name] # repeat this to emphasize the cycle here + push!(deps, modkey.name) # repeat this to emphasize the cycle here + others = Set{String}() for each in package_locks # list the rest of the packages being loaded too if each[2][1] === task other = each[1].name - other == modkey.name || other == pkgid.name || push!(others, other) + other == modkey.name || push!(others, other) end end + # remove duplicates from others already in deps + for dep in deps + delete!(others, dep) + end msg = sprint(deps, others) do io, deps, others print(io, "deadlock detected in loading ") - join(io, deps, " -> ") - print(io, " -> ") - join(io, others, " && ") + join(io, deps, " using ") + if !isempty(others) + print(io, " (while loading ") + join(io, others, " and ") + print(io, ")") + end end throw(ConcurrencyViolationError(msg)) end @@ -2383,6 +2388,10 @@ function __require(into::Module, mod::Symbol) error("`using/import $mod` outside of a Module detected. Importing a package outside of a module \ is not allowed during package precompilation.") end + topmod = moduleroot(into) + if nameof(topmod) === mod + return topmod + end @lock require_lock begin LOADING_CACHE[] = LoadingCache() try @@ -2491,10 +2500,7 @@ function _require_prelocked(uuidkey::PkgId, env=nothing) try toplevel_load[] = false m = __require_prelocked(uuidkey, env) - if m === nothing - error("package `$(uuidkey.name)` did not define the expected \ - module `$(uuidkey.name)`, check for typos in package module name") - end + m isa Module || check_package_module_loaded_error(uuidkey) finally toplevel_load[] = last end_loading(uuidkey, m) @@ -2984,6 +2990,9 @@ const newly_inferred = CodeInstance[] function include_package_for_output(pkg::PkgId, input::String, depot_path::Vector{String}, dl_load_path::Vector{String}, load_path::Vector{String}, concrete_deps::typeof(_concrete_dependencies), source::Union{Nothing,String}) + @lock require_lock begin + m = start_loading(pkg, UInt128(0), false) + @assert m === nothing append!(empty!(Base.DEPOT_PATH), depot_path) append!(empty!(Base.DL_LOAD_PATH), dl_load_path) append!(empty!(Base.LOAD_PATH), load_path) @@ -2992,6 +3001,8 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto Base._track_dependencies[] = true get!(Base.PkgOrigin, Base.pkgorigins, pkg).path = input append!(empty!(Base._concrete_dependencies), concrete_deps) + end + uuid_tuple = pkg.uuid === nothing ? (UInt64(0), UInt64(0)) : convert(NTuple{2, UInt64}, pkg.uuid) ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Base.__toplevel__, uuid_tuple) @@ -3010,21 +3021,22 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto ccall(:jl_set_newly_inferred, Cvoid, (Any,), nothing) end # check that the package defined the expected module so we can give a nice error message if not - Base.check_package_module_loaded(pkg) + m = maybe_root_module(pkg) + m isa Module || check_package_module_loaded_error(pkg) # Re-populate the runtime's newly-inferred array, which will be included # in the output. We removed it above to avoid including any code we may # have compiled for error handling and validation. ccall(:jl_set_newly_inferred, Cvoid, (Any,), newly_inferred) + @lock require_lock end_loading(pkg, m) + # insert_extension_triggers(pkg) + # run_package_callbacks(pkg) end -function check_package_module_loaded(pkg::PkgId) - if !haskey(Base.loaded_modules, pkg) - # match compilecache error type for non-125 errors - error("$(repr("text/plain", pkg)) did not define the expected module `$(pkg.name)`, \ - check for typos in package module name") - end - return nothing +function check_package_module_loaded_error(pkg) + # match compilecache error type for non-125 errors + error("package `$(pkg.name)` did not define the expected \ + module `$(pkg.name)`, check for typos in package module name") end # protects against PkgId and UUID being imported and losing Base prefix @@ -3100,7 +3112,6 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: Base.track_nested_precomp($(_pkg_str(vcat(Base.precompilation_stack, pkg)))) Base.loadable_extensions = $(_pkg_str(loadable_exts)) Base.precompiling_extension = $(loading_extension) - Base.precompilation_target = $(_pkg_str(pkg)) Base.include_package_for_output($(_pkg_str(pkg)), $(repr(abspath(input))), $(repr(depot_path)), $(repr(dl_load_path)), $(repr(load_path)), $(_pkg_str(concrete_deps)), $(repr(source_path(nothing)))) """) diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index b4d1cde9527be..a6c66bbd14f77 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -371,7 +371,7 @@ There are three important standard modules: Modules can contain *submodules*, nesting the same syntax `module ... end`. They can be used to introduce separate namespaces, which can be helpful for organizing complex codebases. Note that each `module` introduces its own [scope](@ref scope-of-variables), so submodules do not automatically “inherit” names from their parent. -It is recommended that submodules refer to other modules within the enclosing parent module (including the latter) using *relative module qualifiers* in `using` and `import` statements. A relative module qualifier starts with a period (`.`), which corresponds to the current module, and each successive `.` leads to the parent of the current module. This should be followed by modules if necessary, and eventually the actual name to access, all separated by `.`s. +It is recommended that submodules refer to other modules within the enclosing parent module (including the latter) using *relative module qualifiers* in `using` and `import` statements. A relative module qualifier starts with a period (`.`), which corresponds to the current module, and each successive `.` leads to the parent of the current module. This should be followed by modules if necessary, and eventually the actual name to access, all separated by `.`s. As a special case, however, referring to the module root can be written without `.`, avoiding the need to count the depth to reach that module. Consider the following example, where the submodule `SubA` defines a function, which is then extended in its “sibling” module: @@ -386,6 +386,7 @@ julia> module ParentModule export add_D # export it from ParentModule too module SubB import ..SubA: add_D # relative path for a “sibling” module + # import ParentModule.SubA: add_D # when in a package, such as when this is loaded by using or import, this would be equivalent to the previous import, but not at the REPL struct Infinity end add_D(x::Infinity) = x end @@ -393,12 +394,16 @@ julia> module ParentModule ``` -You may see code in packages, which, in a similar situation, uses +You may see code in packages, which, in a similar situation, uses import without the `.`: +```jldoctest +julia> import ParentModule.SubA: add_D +ERROR: ArgumentError: Package ParentModule not found in current path. +``` +However, since this operates through [code loading](@ref code-loading), it only works if `ParentModule` is in a package in a file. If `ParentModule` was defined at the REPL, it is necessary to use use relative paths: ```jldoctest module_manual julia> import .ParentModule.SubA: add_D ``` -However, this operates through [code loading](@ref code-loading), and thus only works if `ParentModule` is in a package. It is better to use relative paths. Note that the order of definitions also matters if you are evaluating values. Consider @@ -491,8 +496,12 @@ In particular, if you define a `function __init__()` in a module, then Julia wil immediately *after* the module is loaded (e.g., by `import`, `using`, or `require`) at runtime for the *first* time (i.e., `__init__` is only called once, and only after all statements in the module have been executed). Because it is called after the module is fully imported, any submodules -or other imported modules have their `__init__` functions called *before* the `__init__` of the -enclosing module. +or other imported modules have their `__init__` functions called *before* the `__init__` of +the enclosing module. This is also synchronized across threads, so that code can safely rely upon +this ordering of effects, such that all `__init__` will have run, in dependency ordering, +before the `using` result is completed. They may run concurrently with other `__init__` +methods which are not dependencies however, so be careful when accessing any shared state +outside the current module to use locks when needed. Two typical uses of `__init__` are calling runtime initialization functions of external C libraries and initializing global constants that involve pointers returned by external libraries. For example, @@ -524,17 +533,6 @@ pointer value must be called at runtime for precompilation to work ([`Ptr`](@ref null pointers unless they are hidden inside an [`isbits`](@ref) object). This includes the return values of the Julia functions [`@cfunction`](@ref) and [`pointer`](@ref). -Dictionary and set types, or in general anything that depends on the output of a `hash(key)` method, -are a trickier case. In the common case where the keys are numbers, strings, symbols, ranges, -`Expr`, or compositions of these types (via arrays, tuples, sets, pairs, etc.) they are safe to -precompile. However, for a few other key types, such as `Function` or `DataType` and generic -user-defined types where you haven't defined a `hash` method, the fallback `hash` method depends -on the memory address of the object (via its `objectid`) and hence may change from run to run. -If you have one of these key types, or if you aren't sure, to be safe you can initialize this -dictionary from within your `__init__` function. Alternatively, you can use the [`IdDict`](@ref) -dictionary type, which is specially handled by precompilation so that it is safe to initialize -at compile-time. - When using precompilation, it is important to keep a clear sense of the distinction between the compilation phase and the execution phase. In this mode, it will often be much more clearly apparent that Julia is a compiler which allows execution of arbitrary Julia code, not a standalone interpreter diff --git a/test/loading.jl b/test/loading.jl index be8f08b4bfe22..d12cd2769ef1d 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1345,9 +1345,9 @@ module loaded_pkgid4 end end wait(e) reset(e) - @test_throws(ConcurrencyViolationError("deadlock detected in loading pkgid3 -> pkgid2 -> pkgid1 -> pkgid3 && pkgid4"), + @test_throws(ConcurrencyViolationError("deadlock detected in loading pkgid3 using pkgid2 using pkgid1 using pkgid3 (while loading pkgid4)"), @lock Base.require_lock Base.start_loading(pkid3, build_id, false)).value # try using pkgid3 - @test_throws(ConcurrencyViolationError("deadlock detected in loading pkgid4 -> pkgid4 && pkgid1"), + @test_throws(ConcurrencyViolationError("deadlock detected in loading pkgid4 using pkgid4 (while loading pkgid1)"), @lock Base.require_lock Base.start_loading(pkid4, build_id, false)).value # try using pkgid4 @lock Base.require_lock Base.end_loading(pkid1, loaded_pkgid1) # end @lock Base.require_lock Base.end_loading(pkid4, loaded_pkgid4) # end diff --git a/test/precompile.jl b/test/precompile.jl index e246133abf106..7c5c63a277e27 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1633,25 +1633,28 @@ precompile_test_harness("Issue #26028") do load_path """ module Foo26028 module Bar26028 + using Foo26028: Foo26028 as InnerFoo1 + using ..Foo26028: Foo26028 as InnerFoo2 x = 0 y = 0 end function __init__() - include(joinpath(@__DIR__, "Baz26028.jl")) - end + Baz = @eval module Baz26028 + using Test + public @test_throws + import Foo26028.Bar26028.y as y1 + import ..Foo26028.Bar26028.y as y2 + end + @eval Base \$Baz.@test_throws(ConcurrencyViolationError("deadlock detected in loading Foo26028 using Foo26028"), + import Foo26028.Bar26028.x) end - """) - write(joinpath(load_path, "Baz26028.jl"), - """ - module Baz26028 - using Test - @test_throws(ConcurrencyViolationError("deadlock detected in loading Foo26028 -> Foo26028"), - @eval import Foo26028.Bar26028.x) - import ..Foo26028.Bar26028.y end """) Base.compilecache(Base.PkgId("Foo26028")) @test_nowarn @eval using Foo26028 + invokelatest() do + @test Foo26028 === Foo26028.Bar26028.InnerFoo1 === Foo26028.Bar26028.InnerFoo2 + end end precompile_test_harness("Issue #29936") do load_path @@ -2265,7 +2268,7 @@ precompile_test_harness("No package module") do load_path """) @test_throws r"Failed to precompile NoModule" Base.compilecache(Base.identify_package("NoModule"), io, io) @test occursin( - "NoModule [top-level] did not define the expected module `NoModule`, check for typos in package module name", + "package `NoModule` did not define the expected module `NoModule`, check for typos in package module name", String(take!(io))) @@ -2277,7 +2280,7 @@ precompile_test_harness("No package module") do load_path """) @test_throws r"Failed to precompile WrongModuleName" Base.compilecache(Base.identify_package("WrongModuleName"), io, io) @test occursin( - "WrongModuleName [top-level] did not define the expected module `WrongModuleName`, check for typos in package module name", + "package `WrongModuleName` did not define the expected module `WrongModuleName`, check for typos in package module name", String(take!(io))) @@ -2398,4 +2401,19 @@ precompile_test_harness("MainImportDisallow") do load_path end end +precompile_test_harness("Package top-level load itself") do load_path + write(joinpath(load_path, "UsingSelf.jl"), + """ + __precompile__(false) + module UsingSelf + using UsingSelf + x = 3 + end + """) + @eval using UsingSelf + invokelatest() do + @test UsingSelf.x == 3 + end +end + finish_precompile_test!() From 69d7a98344f9ece0329d4309764a091015de6a5b Mon Sep 17 00:00:00 2001 From: Neven Sajko <4944410+nsajko@users.noreply.github.com> Date: Fri, 21 Mar 2025 18:38:20 +0100 Subject: [PATCH 69/74] iobuffer: copyline: type assert `Int` to prevent invalidation (#57848) Should prevent invalidation stemming from `MethodInstance`s for * `readline()` * `readlines()` * `Base.run_fallback_repl(::Bool)` * `Base.repl_main(::Any)` * `Base._start()` In each case the issue is the use of the nonconcretely-typed global variable `stdin::IO`. (cherry picked from commit eed18bdf706b7aab15b12f3ba0588e8fafcd4930) --- base/iobuffer.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base/iobuffer.jl b/base/iobuffer.jl index 7e309b9ad586c..144b0a20568e9 100644 --- a/base/iobuffer.jl +++ b/base/iobuffer.jl @@ -643,7 +643,7 @@ function _copyline(out::IO, io::GenericIOBuffer; keep::Bool=false) data = view(io.data, io.ptr:io.size) # note: findfirst + copyto! is much faster than a single loop # except for nout ≲ 20. A single loop is 2x faster for nout=5. - nout = nread = something(findfirst(==(0x0a), data), length(data)) + nout = nread = something(findfirst(==(0x0a), data), length(data))::Int if !keep && nout > 0 && data[nout] == 0x0a nout -= 1 nout > 0 && data[nout] == 0x0d && (nout -= 1) From a60a9788eca8876dc7f8a38dd44063134dad8ae5 Mon Sep 17 00:00:00 2001 From: Sam Schweigel <33556084+xal-0@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:52:54 -0700 Subject: [PATCH 70/74] Make remaining float intrinsics require float arguments (#57398) The `fptrunc`/`fpext` intrinsics were modified in #57160 to throw on non-float arguments. - The arithmetic and math float intrinsics now require all their arguments to be floats - `fptosi`/`fptoui` require their source to be a float - `sitofp`/`uitofp` require their destination type to be a float Also fixes #57384. (cherry picked from commit 0bcc9cd928e8f749b52dbdb04db29bc8fc0f329b) --- Compiler/src/tfuncs.jl | 54 ++++++++++++++++++-- Compiler/test/effects.jl | 26 ++++++++++ src/APInt-C.cpp | 4 +- src/intrinsics.cpp | 35 ++++++++----- src/runtime_intrinsics.c | 103 +++++++++++++++------------------------ test/intrinsics.jl | 15 +++++- 6 files changed, 155 insertions(+), 82 deletions(-) diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index cacee7a7d29ac..1456ac977d9f9 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -2452,6 +2452,43 @@ const _SPECIAL_BUILTINS = Any[ Core._apply_iterate, ] +# Intrinsics that require all arguments to be floats +const _FLOAT_INTRINSICS = Any[ + Intrinsics.neg_float, + Intrinsics.add_float, + Intrinsics.sub_float, + Intrinsics.mul_float, + Intrinsics.div_float, + Intrinsics.min_float, + Intrinsics.max_float, + Intrinsics.fma_float, + Intrinsics.muladd_float, + Intrinsics.neg_float_fast, + Intrinsics.add_float_fast, + Intrinsics.sub_float_fast, + Intrinsics.mul_float_fast, + Intrinsics.div_float_fast, + Intrinsics.min_float_fast, + Intrinsics.max_float_fast, + Intrinsics.eq_float, + Intrinsics.ne_float, + Intrinsics.lt_float, + Intrinsics.le_float, + Intrinsics.eq_float_fast, + Intrinsics.ne_float_fast, + Intrinsics.lt_float_fast, + Intrinsics.le_float_fast, + Intrinsics.fpiseq, + Intrinsics.abs_float, + Intrinsics.copysign_float, + Intrinsics.ceil_llvm, + Intrinsics.floor_llvm, + Intrinsics.trunc_llvm, + Intrinsics.rint_llvm, + Intrinsics.sqrt_llvm, + Intrinsics.sqrt_llvm_fast +] + # Types compatible with fpext/fptrunc const CORE_FLOAT_TYPES = Union{Core.BFloat16, Float16, Float32, Float64} @@ -2869,7 +2906,8 @@ function intrinsic_exct(𝕃::AbstractLattice, f::IntrinsicFunction, argtypes::V return ErrorException end - # fpext and fptrunc have further restrictions on the allowed types. + # fpext, fptrunc, fptoui, fptosi, uitofp, and sitofp have further + # restrictions on the allowed types. if f === Intrinsics.fpext && !(ty <: CORE_FLOAT_TYPES && xty <: CORE_FLOAT_TYPES && Core.sizeof(ty) > Core.sizeof(xty)) return ErrorException @@ -2878,6 +2916,12 @@ function intrinsic_exct(𝕃::AbstractLattice, f::IntrinsicFunction, argtypes::V !(ty <: CORE_FLOAT_TYPES && xty <: CORE_FLOAT_TYPES && Core.sizeof(ty) < Core.sizeof(xty)) return ErrorException end + if (f === Intrinsics.fptoui || f === Intrinsics.fptosi) && !(xty <: CORE_FLOAT_TYPES) + return ErrorException + end + if (f === Intrinsics.uitofp || f === Intrinsics.sitofp) && !(ty <: CORE_FLOAT_TYPES) + return ErrorException + end return Union{} end @@ -2890,11 +2934,15 @@ function intrinsic_exct(𝕃::AbstractLattice, f::IntrinsicFunction, argtypes::V return Union{} end - # The remaining intrinsics are math/bits/comparison intrinsics. They work on all - # primitive types of the same type. + # The remaining intrinsics are math/bits/comparison intrinsics. + # All the non-floating point intrinsics work on primitive values of the same type. isshift = f === shl_int || f === lshr_int || f === ashr_int argtype1 = widenconst(argtypes[1]) isprimitivetype(argtype1) || return ErrorException + if contains_is(_FLOAT_INTRINSICS, f) + argtype1 <: CORE_FLOAT_TYPES || return ErrorException + end + for i = 2:length(argtypes) argtype = widenconst(argtypes[i]) if isshift ? !isprimitivetype(argtype) : argtype !== argtype1 diff --git a/Compiler/test/effects.jl b/Compiler/test/effects.jl index d9ac97f36e181..031b7bebaf7a4 100644 --- a/Compiler/test/effects.jl +++ b/Compiler/test/effects.jl @@ -1401,3 +1401,29 @@ end == Compiler.EFFECTS_UNKNOWN @test !Compiler.intrinsic_nothrow(Core.Intrinsics.fpext, Any[Type{Float32}, Float64]) @test !Compiler.intrinsic_nothrow(Core.Intrinsics.fpext, Any[Type{Int32}, Float16]) @test !Compiler.intrinsic_nothrow(Core.Intrinsics.fpext, Any[Type{Float32}, Int16]) + +# Float intrinsics require float arguments +@test Base.infer_effects((Int16,)) do x + return Core.Intrinsics.abs_float(x) +end |> !Compiler.is_nothrow +@test Base.infer_effects((Int32, Int32)) do x, y + return Core.Intrinsics.add_float(x, y) +end |> !Compiler.is_nothrow +@test Base.infer_effects((Int32, Int32)) do x, y + return Core.Intrinsics.add_float(x, y) +end |> !Compiler.is_nothrow +@test Base.infer_effects((Int64, Int64, Int64)) do x, y, z + return Core.Intrinsics.fma_float(x, y, z) +end |> !Compiler.is_nothrow +@test Base.infer_effects((Int64,)) do x + return Core.Intrinsics.fptoui(UInt32, x) +end |> !Compiler.is_nothrow +@test Base.infer_effects((Int64,)) do x + return Core.Intrinsics.fptosi(Int32, x) +end |> !Compiler.is_nothrow +@test Base.infer_effects((Int64,)) do x + return Core.Intrinsics.sitofp(Int64, x) +end |> !Compiler.is_nothrow +@test Base.infer_effects((UInt64,)) do x + return Core.Intrinsics.uitofp(Int64, x) +end |> !Compiler.is_nothrow diff --git a/src/APInt-C.cpp b/src/APInt-C.cpp index 86b0bdb27638b..64b87a1096d44 100644 --- a/src/APInt-C.cpp +++ b/src/APInt-C.cpp @@ -321,7 +321,7 @@ void LLVMFPtoInt(jl_datatype_t *ty, void *pa, jl_datatype_t *oty, integerPart *p Val = julia_bfloat_to_float(*(uint16_t*)pa); else if (ty == jl_float32_type) Val = *(float*)pa; - else if (jl_float64_type) + else if (ty == jl_float64_type) Val = *(double*)pa; else jl_error("FPtoSI: runtime floating point intrinsics are not implemented for bit sizes other than 16, 32 and 64"); @@ -352,7 +352,7 @@ void LLVMFPtoInt(jl_datatype_t *ty, void *pa, jl_datatype_t *oty, integerPart *p else { APFloat a(Val); bool isVeryExact; - APFloat::roundingMode rounding_mode = APFloat::rmNearestTiesToEven; + APFloat::roundingMode rounding_mode = RoundingMode::TowardZero; unsigned nbytes = alignTo(onumbits, integerPartWidth) / host_char_bit; integerPart *parts = (integerPart*)alloca(nbytes); APFloat::opStatus status = a.convertToInteger(MutableArrayRef(parts, nbytes), onumbits, isSigned, rounding_mode, &isVeryExact); diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index 4006397d08ea1..d65c2ec5afcc8 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -676,17 +676,23 @@ static jl_cgval_t generic_cast( Type *to = bitstype_to_llvm((jl_value_t*)jlto, ctx.builder.getContext(), true); Type *vt = bitstype_to_llvm(v.typ, ctx.builder.getContext(), true); - // fptrunc fpext depend on the specific floating point format to work - // correctly, and so do not pun their argument types. + // fptrunc and fpext depend on the specific floating point + // format to work correctly, and so do not pun their argument types. if (!(f == fpext || f == fptrunc)) { - if (toint) - to = INTT(to, DL); - else - to = FLOATT(to); - if (fromint) - vt = INTT(vt, DL); - else - vt = FLOATT(vt); + // uitofp/sitofp require a specific float type argument + if (!(f == uitofp || f == sitofp)){ + if (toint) + to = INTT(to, DL); + else + to = FLOATT(to); + } + // fptoui/fptosi require a specific float value argument + if (!(f == fptoui || f == fptosi)) { + if (fromint) + vt = INTT(vt, DL); + else + vt = FLOATT(vt); + } } if (!to || !vt) @@ -1428,10 +1434,13 @@ static jl_cgval_t emit_intrinsic(jl_codectx_t &ctx, intrinsic f, jl_value_t **ar if (!jl_is_primitivetype(xinfo.typ)) return emit_runtime_call(ctx, f, argv, nargs); Type *xtyp = bitstype_to_llvm(xinfo.typ, ctx.builder.getContext(), true); - if (float_func()[f]) - xtyp = FLOATT(xtyp); - else + if (float_func()[f]) { + if (!xtyp->isFloatingPointTy()) + return emit_runtime_call(ctx, f, argv, nargs); + } + else { xtyp = INTT(xtyp, DL); + } if (!xtyp) return emit_runtime_call(ctx, f, argv, nargs); ////Bool are required to be in the range [0,1] diff --git a/src/runtime_intrinsics.c b/src/runtime_intrinsics.c index 09f55f1d1f1d8..2671bebfd7f55 100644 --- a/src/runtime_intrinsics.c +++ b/src/runtime_intrinsics.c @@ -1073,31 +1073,26 @@ typedef void (fintrinsic_op1)(unsigned, jl_value_t*, void*, void*); static inline jl_value_t *jl_fintrinsic_1(jl_value_t *ty, jl_value_t *a, const char *name, fintrinsic_op1 *bfloatop, fintrinsic_op1 *halfop, fintrinsic_op1 *floatop, fintrinsic_op1 *doubleop) { jl_task_t *ct = jl_current_task; - if (!jl_is_primitivetype(jl_typeof(a))) + jl_datatype_t *aty = (jl_datatype_t *)jl_typeof(a); + if (!jl_is_primitivetype(aty)) jl_errorf("%s: value is not a primitive type", name); if (!jl_is_primitivetype(ty)) jl_errorf("%s: type is not a primitive type", name); unsigned sz2 = jl_datatype_size(ty); jl_value_t *newv = jl_gc_alloc(ct->ptls, sz2, ty); void *pa = jl_data_ptr(a), *pr = jl_data_ptr(newv); - unsigned sz = jl_datatype_size(jl_typeof(a)); - switch (sz) { - /* choose the right size c-type operation based on the input */ - case 2: - if (jl_typeof(a) == (jl_value_t*)jl_float16_type) - halfop(sz2 * host_char_bit, ty, pa, pr); - else /*if (jl_typeof(a) == (jl_value_t*)jl_bfloat16_type)*/ - bfloatop(sz2 * host_char_bit, ty, pa, pr); - break; - case 4: + + if (aty == jl_float16_type) + halfop(sz2 * host_char_bit, ty, pa, pr); + else if (aty == jl_bfloat16_type) + bfloatop(sz2 * host_char_bit, ty, pa, pr); + else if (aty == jl_float32_type) floatop(sz2 * host_char_bit, ty, pa, pr); - break; - case 8: + else if (aty == jl_float64_type) doubleop(sz2 * host_char_bit, ty, pa, pr); - break; - default: - jl_errorf("%s: runtime floating point intrinsics are not implemented for bit sizes other than 16, 32 and 64", name); - } + else + jl_errorf("%s: runtime floating point intrinsics require both arguments to be Float16, BFloat16, Float32, or Float64", name); + return newv; } @@ -1273,6 +1268,7 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ { \ jl_task_t *ct = jl_current_task; \ jl_value_t *ty = jl_typeof(a); \ + jl_datatype_t *aty = (jl_datatype_t *)ty; \ if (jl_typeof(b) != ty) \ jl_error(#name ": types of a and b must match"); \ if (!jl_is_primitivetype(ty)) \ @@ -1280,23 +1276,16 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ int sz = jl_datatype_size(ty); \ jl_value_t *newv = jl_gc_alloc(ct->ptls, sz, ty); \ void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b), *pr = jl_data_ptr(newv); \ - switch (sz) { \ - /* choose the right size c-type operation */ \ - case 2: \ - if ((jl_datatype_t*)ty == jl_float16_type) \ - jl_##name##16(16, pa, pb, pr); \ - else /*if ((jl_datatype_t*)ty == jl_bfloat16_type)*/ \ - jl_##name##bf16(16, pa, pb, pr); \ - break; \ - case 4: \ + if (aty == jl_float16_type) \ + jl_##name##16(16, pa, pb, pr); \ + else if (aty == jl_bfloat16_type) \ + jl_##name##bf16(16, pa, pb, pr); \ + else if (aty == jl_float32_type) \ jl_##name##32(32, pa, pb, pr); \ - break; \ - case 8: \ + else if (aty == jl_float64_type) \ jl_##name##64(64, pa, pb, pr); \ - break; \ - default: \ - jl_error(#name ": runtime floating point intrinsics are not implemented for bit sizes other than 16, 32 and 64"); \ - } \ + else \ + jl_error(#name ": runtime floating point intrinsics require both arguments to be Float16, BFloat16, Float32, or Float64"); \ return newv; \ } @@ -1308,30 +1297,24 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b) \ { \ jl_value_t *ty = jl_typeof(a); \ + jl_datatype_t *aty = (jl_datatype_t *)ty; \ if (jl_typeof(b) != ty) \ jl_error(#name ": types of a and b must match"); \ if (!jl_is_primitivetype(ty)) \ jl_error(#name ": values are not primitive types"); \ void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b); \ - int sz = jl_datatype_size(ty); \ int cmp; \ - switch (sz) { \ - /* choose the right size c-type operation */ \ - case 2: \ - if ((jl_datatype_t*)ty == jl_float16_type) \ - cmp = jl_##name##16(16, pa, pb); \ - else /*if ((jl_datatype_t*)ty == jl_bfloat16_type)*/ \ - cmp = jl_##name##bf16(16, pa, pb); \ - break; \ - case 4: \ + if (aty == jl_float16_type) \ + cmp = jl_##name##16(16, pa, pb); \ + else if (aty == jl_bfloat16_type) \ + cmp = jl_##name##bf16(16, pa, pb); \ + else if (aty == jl_float32_type) \ cmp = jl_##name##32(32, pa, pb); \ - break; \ - case 8: \ + else if (aty == jl_float64_type) \ cmp = jl_##name##64(64, pa, pb); \ - break; \ - default: \ - jl_error(#name ": runtime floating point intrinsics are not implemented for bit sizes other than 32 and 64"); \ - } \ + else \ + jl_error(#name ": runtime floating point intrinsics require both arguments to be Float16, BFloat16, Float32, or Float64"); \ + \ return cmp ? jl_true : jl_false; \ } @@ -1344,6 +1327,7 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b, jl_value_t *c) { \ jl_task_t *ct = jl_current_task; \ jl_value_t *ty = jl_typeof(a); \ + jl_datatype_t *aty = (jl_datatype_t *)ty; \ if (jl_typeof(b) != ty || jl_typeof(c) != ty) \ jl_error(#name ": types of a, b, and c must match"); \ if (!jl_is_primitivetype(ty)) \ @@ -1351,23 +1335,16 @@ JL_DLLEXPORT jl_value_t *jl_##name(jl_value_t *a, jl_value_t *b, jl_value_t *c) int sz = jl_datatype_size(ty); \ jl_value_t *newv = jl_gc_alloc(ct->ptls, sz, ty); \ void *pa = jl_data_ptr(a), *pb = jl_data_ptr(b), *pc = jl_data_ptr(c), *pr = jl_data_ptr(newv); \ - switch (sz) { \ - /* choose the right size c-type operation */ \ - case 2: \ - if ((jl_datatype_t*)ty == jl_float16_type) \ + if (aty == jl_float16_type) \ jl_##name##16(16, pa, pb, pc, pr); \ - else /*if ((jl_datatype_t*)ty == jl_bfloat16_type)*/ \ + else if (aty == jl_bfloat16_type) \ jl_##name##bf16(16, pa, pb, pc, pr); \ - break; \ - case 4: \ + else if (aty == jl_float32_type) \ jl_##name##32(32, pa, pb, pc, pr); \ - break; \ - case 8: \ + else if (aty == jl_float64_type) \ jl_##name##64(64, pa, pb, pc, pr); \ - break; \ - default: \ - jl_error(#name ": runtime floating point intrinsics are not implemented for bit sizes other than 16, 32 and 64"); \ - } \ + else \ + jl_error(#name ": runtime floating point intrinsics require both arguments to be Float16, BFloat16, Float32, or Float64"); \ return newv; \ } @@ -1661,7 +1638,7 @@ static inline void fptrunc(jl_datatype_t *aty, void *pa, jl_datatype_t *ty, void fptrunc_convert(float64, bfloat16); fptrunc_convert(float64, float32); else - jl_error("fptrunc: runtime floating point intrinsics are not implemented for bit sizes other than 16, 32 and 64"); + jl_error("fptrunc: runtime floating point intrinsics require both arguments to be Float16, BFloat16, Float32, or Float64"); #undef fptrunc_convert } @@ -1685,7 +1662,7 @@ static inline void fpext(jl_datatype_t *aty, void *pa, jl_datatype_t *ty, void * fpext_convert(bfloat16, float64); fpext_convert(float32, float64); else - jl_error("fptrunc: runtime floating point intrinsics are not implemented for bit sizes other than 16, 32 and 64"); + jl_error("fptrunc: runtime floating point intrinsics require both arguments to be Float16, BFloat16, Float32, or Float64"); #undef fpext_convert } diff --git a/test/intrinsics.jl b/test/intrinsics.jl index ee1da6d7a45ae..5e18c1fb3672a 100644 --- a/test/intrinsics.jl +++ b/test/intrinsics.jl @@ -42,6 +42,8 @@ truncbool(u) = reinterpret(UInt8, reinterpret(Bool, u)) @test_throws ErrorException("SExt: output bitsize must be > input bitsize") Core.Intrinsics.sext_int(Int8, 0x0000) @test_throws ErrorException("Trunc: output bitsize must be < input bitsize") Core.Intrinsics.trunc_int(Int8, 0x00) @test_throws ErrorException("Trunc: output bitsize must be < input bitsize") Core.Intrinsics.trunc_int(Int16, 0x00) + + @test_throws ErrorException("add_float: runtime floating point intrinsics require both arguments to be Float16, BFloat16, Float32, or Float64") Core.Intrinsics.add_float(1, 2) end # issue #4581 @@ -92,7 +94,7 @@ compiled_addi(x, y) = Core.Intrinsics.add_int(x, y) @test compiled_addi(true, true) === false compiled_addf(x, y) = Core.Intrinsics.add_float(x, y) -@test compiled_addf(C_NULL, C_NULL) === C_NULL +@test_throws ErrorException compiled_addf(C_NULL, C_NULL) @test_throws ErrorException compiled_addf(C_NULL, 1) @test compiled_addf(0.5, 5.0e-323) === 0.5 @test_throws ErrorException compiled_addf(im, im) @@ -231,6 +233,8 @@ end # ternary @test_intrinsic Core.Intrinsics.fma_float Float64(3.3) Float64(4.4) Float64(5.5) Float64(20.02) @test_intrinsic Core.Intrinsics.muladd_float Float64(3.3) Float64(4.4) Float64(5.5) Float64(20.02) + @test_intrinsic Core.Intrinsics.fma_float 0x1.0000000000001p0 1.25 0x1p-54 0x1.4000000000002p0 + @test 0x1.0000000000001p0*1.25+0x1p-54 === 0x1.4000000000001p0 # for comparison # boolean @test_intrinsic Core.Intrinsics.eq_float Float64(3.3) Float64(3.3) true @@ -245,6 +249,10 @@ end @test_intrinsic Core.Intrinsics.uitofp Float64 UInt(3) Float64(3.0) @test_intrinsic Core.Intrinsics.fptosi Int Float64(3.3) 3 @test_intrinsic Core.Intrinsics.fptoui UInt Float64(3.3) UInt(3) + + # #57384 + @test_intrinsic Core.Intrinsics.fptosi Int 1.5 1 + @test_intrinsic Core.Intrinsics.fptosi Int128 1.5 Int128(1) end @testset "Float32 intrinsics" begin @@ -265,6 +273,9 @@ end # ternary @test_intrinsic Core.Intrinsics.fma_float Float32(3.3) Float32(4.4) Float32(5.5) Float32(20.02) @test_intrinsic Core.Intrinsics.muladd_float Float32(3.3) Float32(4.4) Float32(5.5) Float32(20.02) + @test_intrinsic Core.Intrinsics.fma_float Float32(0x1.000002p0) 1.25f0 Float32(0x1p-25) Float32(0x1.400004p0) + @test Float32(0x1.000002p0)*1.25f0+Float32(0x1p-25) === Float32(0x1.400002p0) # for comparison + # boolean @test_intrinsic Core.Intrinsics.eq_float Float32(3.3) Float32(3.3) true @@ -357,6 +368,8 @@ end # ternary @test_intrinsic Core.Intrinsics.fma_float Float16(3.3) Float16(4.4) Float16(5.5) Float16(20.02) @test_intrinsic Core.Intrinsics.muladd_float Float16(3.3) Float16(4.4) Float16(5.5) Float16(20.02) + @test_intrinsic Core.Intrinsics.fma_float Float16(0x1.004p0) Float16(1.25) Float16(0x1p-12) Float16(0x1.408p0) + @test Float16(0x1.004p0)*Float16(1.25)+Float16(0x1p-12) === Float16(0x1.404p0) # for comparison # boolean @test_intrinsic Core.Intrinsics.eq_float Float16(3.3) Float16(3.3) true From 1a1442902f98402269e78cddb7575a0d368b6115 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 21 Mar 2025 16:53:38 -0400 Subject: [PATCH 71/74] restore method count after redefinition to hide old definition (#57837) Purely on the external show, since the method does still exist for internals purposes (e.g. method deletion) and is already filtered for inference users (with lim > 0). Close #53814 (cherry picked from commit 35e2886ff4b84060ca1df7930fb855e846476a93) --- Compiler/test/invalidation.jl | 4 ++-- base/runtime_internals.jl | 37 +++++++++++++++++++++++------------ test/core.jl | 2 +- test/reflection.jl | 2 +- 4 files changed, 29 insertions(+), 16 deletions(-) diff --git a/Compiler/test/invalidation.jl b/Compiler/test/invalidation.jl index b77c7677e6987..db9d3ac06048f 100644 --- a/Compiler/test/invalidation.jl +++ b/Compiler/test/invalidation.jl @@ -142,8 +142,8 @@ begin # this redefinition below should invalidate the cache of `pr48932_callee` but not that of `pr48932_caller` pr48932_callee(x) = (print(GLOBAL_BUFFER, x); nothing) - @test length(Base.methods(pr48932_callee)) == 2 - @test Base.only(Base.methods(pr48932_callee, Tuple{Any})) === first(Base.methods(pr48932_callee)) + @test length(Base.methods(pr48932_callee)) == 1 + @test Base.only(Base.methods(pr48932_callee, Tuple{Any})) === only(Base.methods(pr48932_callee)) @test isempty(Base.specializations(Base.only(Base.methods(pr48932_callee, Tuple{Any})))) let mi = only(Base.specializations(Base.only(Base.methods(pr48932_caller)))) # Base.method_instance(pr48932_callee, (Any,)) diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 22fba966679d9..0bfb00b17a630 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -1321,6 +1321,24 @@ function MethodList(mt::Core.MethodTable) return MethodList(ms, mt) end +function matches_to_methods(ms::Array{Any,1}, mt::Core.MethodTable, mod) + # Lack of specialization => a comprehension triggers too many invalidations via _collect, so collect the methods manually + ms = Method[(ms[i]::Core.MethodMatch).method for i in 1:length(ms)] + # Remove shadowed methods with identical type signatures + prev = nothing + filter!(ms) do m + l = prev + repeated = (l isa Method && m.sig == l.sig) + prev = m + return !repeated + end + # Remove methods not part of module (after removing shadowed methods) + mod === nothing || filter!(ms) do m + return parentmodule(m) ∈ mod + end + return MethodList(ms, mt) +end + """ methods(f, [types], [module]) @@ -1328,7 +1346,7 @@ Return the method table for `f`. If `types` is specified, return an array of methods whose types match. If `module` is specified, return an array of methods defined in that module. -A list of modules can also be specified as an array. +A list of modules can also be specified as an array or set. !!! compat "Julia 1.4" At least Julia 1.4 is required for specifying a module. @@ -1336,16 +1354,11 @@ A list of modules can also be specified as an array. See also: [`which`](@ref), [`@which`](@ref Main.InteractiveUtils.@which) and [`methodswith`](@ref Main.InteractiveUtils.methodswith). """ function methods(@nospecialize(f), @nospecialize(t), - mod::Union{Tuple{Module},AbstractArray{Module},Nothing}=nothing) + mod::Union{Tuple{Module},AbstractArray{Module},AbstractSet{Module},Nothing}=nothing) world = get_world_counter() world == typemax(UInt) && error("code reflection cannot be used from generated functions") - # Lack of specialization => a comprehension triggers too many invalidations via _collect, so collect the methods manually - ms = Method[] - for m in _methods(f, t, -1, world)::Vector - m = m::Core.MethodMatch - (mod === nothing || parentmodule(m.method) ∈ mod) && push!(ms, m.method) - end - MethodList(ms, typeof(f).name.mt) + ms = _methods(f, t, -1, world)::Vector{Any} + return matches_to_methods(ms, typeof(f).name.mt, mod) end methods(@nospecialize(f), @nospecialize(t), mod::Module) = methods(f, t, (mod,)) @@ -1355,12 +1368,12 @@ function methods_including_ambiguous(@nospecialize(f), @nospecialize(t)) world == typemax(UInt) && error("code reflection cannot be used from generated functions") min = RefValue{UInt}(typemin(UInt)) max = RefValue{UInt}(typemax(UInt)) - ms = _methods_by_ftype(tt, nothing, -1, world, true, min, max, Ptr{Int32}(C_NULL))::Vector - return MethodList(Method[(m::Core.MethodMatch).method for m in ms], typeof(f).name.mt) + ms = _methods_by_ftype(tt, nothing, -1, world, true, min, max, Ptr{Int32}(C_NULL))::Vector{Any} + return matches_to_methods(ms, typeof(f).name.mt, nothing) end function methods(@nospecialize(f), - mod::Union{Module,AbstractArray{Module},Nothing}=nothing) + mod::Union{Module,AbstractArray{Module},AbstractSet{Module},Nothing}=nothing) # return all matches return methods(f, Tuple{Vararg{Any}}, mod) end diff --git a/test/core.jl b/test/core.jl index 8895d3a7bb86e..9a90e97ccf950 100644 --- a/test/core.jl +++ b/test/core.jl @@ -7691,7 +7691,7 @@ end # issue #31696 foo31696(x::Int8, y::Int8) = 1 foo31696(x::T, y::T) where {T <: Int8} = 2 -@test length(methods(foo31696)) == 2 +@test length(methods(foo31696)) == 1 let T1 = Tuple{Int8}, T2 = Tuple{T} where T<:Int8, a = T1[(1,)], b = T2[(1,)] b .= a @test b[1] == (1,) diff --git a/test/reflection.jl b/test/reflection.jl index a0e9d96f044be..3752bd9c56c88 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -569,7 +569,7 @@ fLargeTable() = 4 fLargeTable(::Union, ::Union) = "a" @test fLargeTable(Union{Int, Missing}, Union{Int, Missing}) == "a" fLargeTable(::Union, ::Union) = "b" -@test length(methods(fLargeTable)) == 206 +@test length(methods(fLargeTable)) == 205 @test fLargeTable(Union{Int, Missing}, Union{Int, Missing}) == "b" # issue #15280 From dd47fcbbdb9ea32bc7e546dceb2dfa278e2a3919 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Fri, 21 Mar 2025 16:55:56 -0400 Subject: [PATCH 72/74] codegen: fix alignment of source in typed_load from a unsafe_load (#57845) The unsafe_load code tries to reuse some of the logic from safe loads, but needs to be careful to override the parts that are not safe with slower versions of them. Similar to the typo-fix from ec3c02af5cd6e63631c7998792e41c679e1c5364 on v1.11-backports. Seen in the IR for code_llvm(optimize=false, raw=true, unsafe_load, (Ptr{Tuple{UInt128}},)) causing LightBSON to fail. Fix https://github.com/ancapdev/LightBSON.jl/issues/37 (cherry picked from commit 36b046dc1137a12b28f3a1e285d385b4b7d1ec89) --- src/ccall.cpp | 2 +- src/cgutils.cpp | 31 +++++++++++++++++-------------- src/codegen.cpp | 17 ++++++++++------- src/intrinsics.cpp | 8 ++++---- 4 files changed, 32 insertions(+), 26 deletions(-) diff --git a/src/ccall.cpp b/src/ccall.cpp index c35979eb85b1d..865278d525384 100644 --- a/src/ccall.cpp +++ b/src/ccall.cpp @@ -547,7 +547,7 @@ static Value *julia_to_native( Align align(julia_alignment(jlto)); Value *slot = emit_static_alloca(ctx, to, align); setName(ctx.emission_context, slot, "native_convert_buffer"); - emit_unbox_store(ctx, jvinfo, slot, ctx.tbaa().tbaa_stack, align); + emit_unbox_store(ctx, jvinfo, slot, ctx.tbaa().tbaa_stack, align, align); return slot; } diff --git a/src/cgutils.cpp b/src/cgutils.cpp index 1e8a1e475e2ee..bd6670c322ef6 100644 --- a/src/cgutils.cpp +++ b/src/cgutils.cpp @@ -301,7 +301,7 @@ static Value *emit_pointer_from_objref(jl_codectx_t &ctx, Value *V) } static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_value_t *jt); -static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, Align alignment, bool isVolatile=false); +static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value* dest, MDNode *tbaa_dest, MaybeAlign align_src, Align align_dst, bool isVolatile=false); static bool type_is_permalloc(jl_value_t *typ) { @@ -1090,7 +1090,7 @@ static void split_value_into(jl_codectx_t &ctx, const jl_cgval_t &x, Align align return; } if (inline_roots_ptr == nullptr) { - emit_unbox_store(ctx, x, dst, ctx.tbaa().tbaa_stack, align_dst, isVolatileStore); + emit_unbox_store(ctx, x, dst, ctx.tbaa().tbaa_stack, align_src, align_dst, isVolatileStore); return; } Value *src = data_pointer(ctx, value_to_pointer(ctx, x)); @@ -1152,7 +1152,7 @@ static void split_value_into(jl_codectx_t &ctx, const jl_cgval_t &x, Align align return; } if (inline_roots.empty()) { - emit_unbox_store(ctx, x, dst, ctx.tbaa().tbaa_stack, align_dst); + emit_unbox_store(ctx, x, dst, ctx.tbaa().tbaa_stack, align_src, align_dst, false); return; } Value *src = data_pointer(ctx, value_to_pointer(ctx, x)); @@ -2351,7 +2351,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, r = boxed(ctx, rhs); } else if (intcast) { - emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, MaybeAlign(), intcast->getAlign()); r = ctx.builder.CreateLoad(realelty, intcast); } else if (aliasscope || Order != AtomicOrdering::NotAtomic || (tracked_pointers && rhs.inline_roots.empty())) { @@ -2389,7 +2389,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } else { assert(Order == AtomicOrdering::NotAtomic && !isboxed && rhs.typ == jltype); - emit_unbox_store(ctx, rhs, ptr, tbaa, Align(alignment)); + emit_unbox_store(ctx, rhs, ptr, tbaa, MaybeAlign(), Align(alignment)); } } else if (isswapfield) { @@ -2438,7 +2438,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } cmp = update_julia_type(ctx, cmp, jltype); if (intcast) { - emit_unbox_store(ctx, cmp, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + emit_unbox_store(ctx, cmp, intcast, ctx.tbaa().tbaa_stack, MaybeAlign(), intcast->getAlign()); Compare = ctx.builder.CreateLoad(realelty, intcast); } else { @@ -2509,7 +2509,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, r = boxed(ctx, rhs); } else if (intcast) { - emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, intcast->getAlign()); + emit_unbox_store(ctx, rhs, intcast, ctx.tbaa().tbaa_stack, MaybeAlign(), intcast->getAlign()); r = ctx.builder.CreateLoad(realelty, intcast); if (!tracked_pointers) // oldval is a slot, so put the oldval back ctx.builder.CreateStore(realCompare, intcast); @@ -2556,7 +2556,7 @@ static jl_cgval_t typed_store(jl_codectx_t &ctx, } else { assert(!isboxed && rhs.typ == jltype); - emit_unbox_store(ctx, rhs, ptr, tbaa, Align(alignment)); + emit_unbox_store(ctx, rhs, ptr, tbaa, MaybeAlign(), Align(alignment)); } ctx.builder.CreateBr(DoneBB); instr = load; @@ -3352,9 +3352,10 @@ static void init_bits_value(jl_codectx_t &ctx, Value *newv, Value *v, MDNode *tb static void init_bits_cgval(jl_codectx_t &ctx, Value *newv, const jl_cgval_t &v) { MDNode *tbaa = jl_is_mutable(v.typ) ? ctx.tbaa().tbaa_mutab : ctx.tbaa().tbaa_immut; - Align newv_align{std::max(julia_alignment(v.typ), (unsigned)sizeof(void*))}; + unsigned alignment = julia_alignment(v.typ); + unsigned newv_align = std::max(alignment, (unsigned)sizeof(void*)); newv = maybe_decay_tracked(ctx, newv); - emit_unbox_store(ctx, v, newv, tbaa, newv_align); + emit_unbox_store(ctx, v, newv, tbaa, Align(alignment), Align(newv_align)); } static jl_value_t *static_constant_instance(const llvm::DataLayout &DL, Constant *constant, jl_value_t *jt) @@ -3808,7 +3809,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con if (jl_is_pointerfree(typ)) { emit_guarded_test(ctx, skip, nullptr, [&] { unsigned alignment = julia_alignment(typ); - emit_unbox_store(ctx, mark_julia_const(ctx, src.constant), dest, tbaa_dst, Align(alignment), isVolatile); + emit_unbox_store(ctx, mark_julia_const(ctx, src.constant), dest, tbaa_dst, Align(alignment), Align(alignment), isVolatile); return nullptr; }); } @@ -3818,7 +3819,7 @@ static void emit_unionmove(jl_codectx_t &ctx, Value *dest, MDNode *tbaa_dst, con if (jl_is_pointerfree(src.typ)) { emit_guarded_test(ctx, skip, nullptr, [&] { unsigned alignment = julia_alignment(src.typ); - emit_unbox_store(ctx, src, dest, tbaa_dst, Align(alignment), isVolatile); + emit_unbox_store(ctx, src, dest, tbaa_dst, Align(alignment), Align(alignment), isVolatile); return nullptr; }); } @@ -4273,6 +4274,8 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg } } else { + Align align_dst(jl_field_align(sty, i)); + Align align_src(julia_alignment(jtype)); if (field_promotable) { fval_info.V->replaceAllUsesWith(dest); cast(fval_info.V)->eraseFromParent(); @@ -4281,10 +4284,10 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg fval = emit_unbox(ctx, fty, fval_info, jtype); } else if (!roots.empty()) { - split_value_into(ctx, fval_info, Align(julia_alignment(jtype)), dest, Align(jl_field_align(sty, i)), jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), roots); + split_value_into(ctx, fval_info, align_src, dest, align_dst, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), roots); } else { - emit_unbox_store(ctx, fval_info, dest, ctx.tbaa().tbaa_stack, Align(jl_field_align(sty, i))); + emit_unbox_store(ctx, fval_info, dest, ctx.tbaa().tbaa_stack, align_src, align_dst); } } if (init_as_value) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 8e524c9d3ba2d..e6c116f029f31 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5731,7 +5731,7 @@ static void emit_vi_assignment_unboxed(jl_codectx_t &ctx, jl_varinfo_t &vi, Valu if (vi.inline_roots) split_value_into(ctx, rval_info, align, vi.value.V, align, jl_aliasinfo_t::fromTBAA(ctx, tbaa), vi.inline_roots, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe), vi.isVolatile); else - emit_unbox_store(ctx, rval_info, vi.value.V, tbaa, align, vi.isVolatile); + emit_unbox_store(ctx, rval_info, vi.value.V, tbaa, align, align, vi.isVolatile); } } } @@ -6947,7 +6947,7 @@ static void emit_specsig_to_specsig( split_value_into(ctx, gf_retval, align, sret, align, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), roots, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_gcframe)); } else { - emit_unbox_store(ctx, gf_retval, sret, ctx.tbaa().tbaa_stack, align); + emit_unbox_store(ctx, gf_retval, sret, ctx.tbaa().tbaa_stack, align, align); } ctx.builder.CreateRetVoid(); break; @@ -8713,11 +8713,12 @@ static jl_llvm_functions_t ++AI; // both specsig (derived) and fptr1 (box) pass this argument as a distinct argument // Load closure world Value *worldaddr = emit_ptrgep(ctx, oc_this, offsetof(jl_opaque_closure_t, world)); + Align alignof_ptr(ctx.types().alignof_ptr); jl_cgval_t closure_world = typed_load(ctx, worldaddr, NULL, (jl_value_t*)jl_long_type, - nullptr, nullptr, false, AtomicOrdering::NotAtomic, false, ctx.types().alignof_ptr.value()); + nullptr, nullptr, false, AtomicOrdering::NotAtomic, false, alignof_ptr.value()); assert(ctx.world_age_at_entry == nullptr); ctx.world_age_at_entry = closure_world.V; // The tls world in a OC is the world of the closure - emit_unbox_store(ctx, closure_world, world_age_field, ctx.tbaa().tbaa_gcframe, ctx.types().alignof_ptr); + emit_unbox_store(ctx, closure_world, world_age_field, ctx.tbaa().tbaa_gcframe, alignof_ptr, alignof_ptr); if (s == jl_unused_sym || vi.value.constant) continue; @@ -9475,7 +9476,7 @@ static jl_llvm_functions_t if (tracked) split_value_into(ctx, typedval, align, dest, align, jl_aliasinfo_t::fromTBAA(ctx, ctx.tbaa().tbaa_stack), mayberoots); else - emit_unbox_store(ctx, typedval, dest, ctx.tbaa().tbaa_stack, align); + emit_unbox_store(ctx, typedval, dest, ctx.tbaa().tbaa_stack, align, align); } return mayberoots; }); @@ -9510,8 +9511,10 @@ static jl_llvm_functions_t else { if (VN) V = Constant::getNullValue(ctx.types().T_prjlvalue); - if (dest) - emit_unbox_store(ctx, val, dest, ctx.tbaa().tbaa_stack, Align(julia_alignment(val.typ))); + if (dest) { + Align align(julia_alignment(val.typ)); + emit_unbox_store(ctx, val, dest, ctx.tbaa().tbaa_stack, align, align); + } RTindex = ConstantInt::get(getInt8Ty(ctx.builder.getContext()), tindex); } } diff --git a/src/intrinsics.cpp b/src/intrinsics.cpp index d65c2ec5afcc8..563ce2fc1270c 100644 --- a/src/intrinsics.cpp +++ b/src/intrinsics.cpp @@ -495,7 +495,7 @@ static Value *emit_unbox(jl_codectx_t &ctx, Type *to, const jl_cgval_t &x, jl_va } // emit code to store a raw value into a destination -static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest, MDNode *tbaa_dest, Align alignment, bool isVolatile) +static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest, MDNode *tbaa_dest, MaybeAlign align_src, Align align_dst, bool isVolatile) { if (x.isghost) { // this can happen when a branch yielding a different type ends @@ -507,14 +507,14 @@ static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest auto dest_ai = jl_aliasinfo_t::fromTBAA(ctx, tbaa_dest); if (!x.inline_roots.empty()) { - recombine_value(ctx, x, dest, dest_ai, alignment, isVolatile); + recombine_value(ctx, x, dest, dest_ai, align_dst, isVolatile); return; } if (!x.ispointer()) { // already unboxed, but sometimes need conversion (e.g. f32 -> i32) assert(x.V); Value *unboxed = zext_struct(ctx, x.V); - StoreInst *store = ctx.builder.CreateAlignedStore(unboxed, dest, alignment); + StoreInst *store = ctx.builder.CreateAlignedStore(unboxed, dest, align_dst); store->setVolatile(isVolatile); dest_ai.decorateInst(store); return; @@ -522,7 +522,7 @@ static void emit_unbox_store(jl_codectx_t &ctx, const jl_cgval_t &x, Value *dest Value *src = data_pointer(ctx, x); auto src_ai = jl_aliasinfo_t::fromTBAA(ctx, x.tbaa); - emit_memcpy(ctx, dest, dest_ai, src, src_ai, jl_datatype_size(x.typ), Align(alignment), Align(julia_alignment(x.typ)), isVolatile); + emit_memcpy(ctx, dest, dest_ai, src, src_ai, jl_datatype_size(x.typ), Align(align_dst), align_src ? *align_src : Align(julia_alignment(x.typ)), isVolatile); } static jl_datatype_t *staticeval_bitstype(const jl_cgval_t &targ) From e4e489a740549322ad3f8ac1860417d367ed3f08 Mon Sep 17 00:00:00 2001 From: Shuhei Kadowaki <40514306+aviatesk@users.noreply.github.com> Date: Sat, 22 Mar 2025 21:03:40 +0900 Subject: [PATCH 73/74] effects: fix effects of atomic pointer operations (#57806) The `_unsetindex!(::GenericMemoryRef)` function was being concretely evaluated (meaning the code was actually being run during inference) by the `REPLInterpreter`. The root cause is that the Base compiler system forgot to mark `atomic_pointerset` as "effect-free". This is simply a bug in the effect system. It didn't cause problems during normal Base compilation because `atomic_pointerset` was correctly marked as "consistent" (concrete evaluation requires both "consistent" and "effect-free"). However, this was still a pretty risky situation. The reason this only caused problems with REPL completion is that the `REPLInterpreter` intentionally ignores the "consistent" requirement under certain conditions to achieve better completion accuracy. This is usually fine, but it relies on "effect-free" being correctly marked. So, when there's a critical bug like this in the effect system, these kinds of dangerous issues can occur. As discussed with Jameson earlier, the effects of atomic pointer operations are not precisely defined. This commit includes the minimal changes necessary to fix JuliaLang/julia#57780, but a more extensive audit is planned for later. - closes JuliaLang/julia#57780 --------- Co-authored-by: Jameson Nash (cherry picked from commit 6a32f7a427789306a0008646900a636a7d91f36e) --- Compiler/src/tfuncs.jl | 55 +++++++++++++++++------------ Compiler/test/effects.jl | 34 ++++++++++++++++++ stdlib/REPL/test/replcompletions.jl | 6 ++++ 3 files changed, 72 insertions(+), 23 deletions(-) diff --git a/Compiler/src/tfuncs.jl b/Compiler/src/tfuncs.jl index 1456ac977d9f9..751baa42c99b0 100644 --- a/Compiler/src/tfuncs.jl +++ b/Compiler/src/tfuncs.jl @@ -2427,11 +2427,8 @@ const _ARGMEM_BUILTINS = Any[ ] const _INCONSISTENT_INTRINSICS = Any[ - Intrinsics.pointerref, # this one is volatile - Intrinsics.sqrt_llvm_fast, # this one may differ at runtime (by a few ulps) - Intrinsics.have_fma, # this one depends on the runtime environment - Intrinsics.cglobal, # cglobal lookup answer changes at runtime - # ... and list fastmath intrinsics: + # all is_pure_intrinsic_infer plus + # ... all the unsound fastmath functions which should have been in is_pure_intrinsic_infer # join(string.("Intrinsics.", sort(filter(endswith("_fast")∘string, names(Core.Intrinsics)))), ",\n") Intrinsics.add_float_fast, Intrinsics.div_float_fast, @@ -2956,36 +2953,48 @@ function intrinsic_nothrow(f::IntrinsicFunction, argtypes::Vector{Any}) return intrinsic_exct(SimpleInferenceLattice.instance, f, argtypes) === Union{} end -# whether `f` is pure for inference -function is_pure_intrinsic_infer(f::IntrinsicFunction) - return !(f === Intrinsics.pointerref || # this one is volatile - f === Intrinsics.pointerset || # this one is never effect-free - f === Intrinsics.llvmcall || # this one is never effect-free - f === Intrinsics.sqrt_llvm_fast || # this one may differ at runtime (by a few ulps) - f === Intrinsics.have_fma || # this one depends on the runtime environment - f === Intrinsics.cglobal) # cglobal lookup answer changes at runtime +function _is_effect_free_infer(f::IntrinsicFunction) + return !(f === Intrinsics.pointerset || + f === Intrinsics.atomic_pointerref || + f === Intrinsics.atomic_pointerset || + f === Intrinsics.atomic_pointerswap || + # f === Intrinsics.atomic_pointermodify || + f === Intrinsics.atomic_pointerreplace || + f === Intrinsics.atomic_fence) end -# whether `f` is effect free if nothrow -function intrinsic_effect_free_if_nothrow(@nospecialize f) - return f === Intrinsics.pointerref || - f === Intrinsics.have_fma || - is_pure_intrinsic_infer(f) +# whether `f` is pure for inference +function is_pure_intrinsic_infer(f::IntrinsicFunction, is_effect_free::Union{Nothing,Bool}=nothing) + if is_effect_free === nothing + is_effect_free = _is_effect_free_infer(f) + end + return is_effect_free && !( + f === Intrinsics.llvmcall || # can do arbitrary things + f === Intrinsics.atomic_pointermodify || # can do arbitrary things + f === Intrinsics.pointerref || # this one is volatile + f === Intrinsics.sqrt_llvm_fast || # this one may differ at runtime (by a few ulps) + f === Intrinsics.have_fma || # this one depends on the runtime environment + f === Intrinsics.cglobal) # cglobal lookup answer changes at runtime end function intrinsic_effects(f::IntrinsicFunction, argtypes::Vector{Any}) if f === Intrinsics.llvmcall # llvmcall can do arbitrary things return Effects() + elseif f === atomic_pointermodify + # atomic_pointermodify has memory effects, plus any effects from the ModifyOpInfo + return Effects() end - if contains_is(_INCONSISTENT_INTRINSICS, f) - consistent = ALWAYS_FALSE - else + is_effect_free = _is_effect_free_infer(f) + effect_free = is_effect_free ? ALWAYS_TRUE : ALWAYS_FALSE + if ((is_pure_intrinsic_infer(f, is_effect_free) && !contains_is(_INCONSISTENT_INTRINSICS, f)) || + f === Intrinsics.pointerset || f === Intrinsics.atomic_pointerset || f === Intrinsics.atomic_fence) consistent = ALWAYS_TRUE + else + consistent = ALWAYS_FALSE end - effect_free = !(f === Intrinsics.pointerset) ? ALWAYS_TRUE : ALWAYS_FALSE nothrow = intrinsic_nothrow(f, argtypes) - inaccessiblememonly = ALWAYS_TRUE + inaccessiblememonly = is_effect_free && !(f === Intrinsics.pointerref) ? ALWAYS_TRUE : ALWAYS_FALSE return Effects(EFFECTS_TOTAL; consistent, effect_free, nothrow, inaccessiblememonly) end diff --git a/Compiler/test/effects.jl b/Compiler/test/effects.jl index 031b7bebaf7a4..ce2f94b8e355a 100644 --- a/Compiler/test/effects.jl +++ b/Compiler/test/effects.jl @@ -1427,3 +1427,37 @@ end |> !Compiler.is_nothrow @test Base.infer_effects((UInt64,)) do x return Core.Intrinsics.uitofp(Int64, x) end |> !Compiler.is_nothrow + +# effects modeling for pointer-related intrinsics +let effects = Base.infer_effects(Core.Intrinsics.pointerref, Tuple{Vararg{Any}}) + @test !Compiler.is_consistent(effects) + @test Compiler.is_effect_free(effects) + @test !Compiler.is_inaccessiblememonly(effects) +end +let effects = Base.infer_effects(Core.Intrinsics.pointerset, Tuple{Vararg{Any}}) + @test Compiler.is_consistent(effects) + @test !Compiler.is_effect_free(effects) +end +# effects modeling for atomic intrinsics +# these functions especially need to be marked !effect_free since they imply synchronization +for atomicfunc = Any[ + Core.Intrinsics.atomic_pointerref, + Core.Intrinsics.atomic_pointerset, + Core.Intrinsics.atomic_pointerswap, + Core.Intrinsics.atomic_pointerreplace, + Core.Intrinsics.atomic_fence] + @test !Compiler.is_effect_free(Base.infer_effects(atomicfunc, Tuple{Vararg{Any}})) +end + +# effects modeling for intrinsics that can do arbitrary things +let effects = Base.infer_effects(Core.Intrinsics.llvmcall, Tuple{Vararg{Any}}) + @test effects == Compiler.Effects() +end +let effects = Base.infer_effects(Core.Intrinsics.atomic_pointermodify, Tuple{Vararg{Any}}) + @test effects == Compiler.Effects() +end + +# JuliaLang/julia#57780 +let effects = Base.infer_effects(Base._unsetindex!, (MemoryRef{String},)) + @test !Compiler.is_effect_free(effects) +end diff --git a/stdlib/REPL/test/replcompletions.jl b/stdlib/REPL/test/replcompletions.jl index 59e994f88945b..d9aa11cf609dd 100644 --- a/stdlib/REPL/test/replcompletions.jl +++ b/stdlib/REPL/test/replcompletions.jl @@ -2484,3 +2484,9 @@ let (c, r, res) = test_complete_context("global xxx::Number = Base.", Main) @test res @test "pi" ∈ c end + +# JuliaLang/julia#57780 +const issue57780 = ["a", "b", "c"] +const issue57780_orig = copy(issue57780) +test_complete_context("empty!(issue57780).", Main) +@test issue57780 == issue57780_orig From ed23a99d5dfbcd647db2c0f1df4bd47b42e9dd1a Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Sat, 22 Mar 2025 02:33:36 -0700 Subject: [PATCH 74/74] bpart: Redesign representation of implicit imports (#57755) When we explicitly import a binding (via either `using M: x` or `import`), the corresponding bpart representation is a direct pointer to the imported binding. This is necessary, because that is the precise semantic representation of the import. However, for implicit imports (those created via `using M` without explicit symbol name), the situation is a bit different. Here, there is no semantic content to the particular binding being imported. Worse, the binding is not necessarily unique. Our current semantics are essentially that we walk the entire import graph (both explicit and implicit edges) and if all reachable terminal nodes are either (i) the same binding or (ii) constant bindings with the same value, the import is allowed. In this, we are supposed to ignore cycles, although the current implementation has some trouble with this (#57638, #57699). If the import succeeds, in the current implementation, we then record an arbitrary implicit import edge as in the BindingPartition. In essence, we are creating a spanning tree of the import graph (formed from both the implicit and explicit import edges). However, dynamic algorithms for spanning tree maintenance are complicated and we didn't implement any. As a result, it is possible for later edge additions to accidentally introduce cycles (causing #57699). An additional problem, even if we could keep a consistent spanning tree, is that the spanning tree is not unique. In practice, this is not supposed to be observable, but it is still odd to have a non-unique representation for a particular set of imports. That said, we don't really need the spanning tree. The only place that actually uses it is `which(::Module, ::Symbol)` which is not performance sensitive and arguably a bad API for the above reason that the answer is ill-defined. With all these problems, let's just get rid of the spanning tree all together - as mentioned we don't really need it. Instead, we split the PARTITION_KIND_IMPLICIT into two, one for each of the two cases (importing a global binding, or importing a particular constant value). This is actually a more efficient implementation for the common case of needing to follow a lookup - we no longer need to follow all the import edges. In exchange, more work needs to be done during binding replacement to re-scan the entire import graph. This is probably the right trade-off though, since binding replacement is already a slow path. Fixes #57638, fixes #57699, fixes #57641, fixes #57700, fixes #57343 (cherry picked from commit 60a9f6992d82789e23a1e71f1e3c7402b1483b4f) --- Compiler/src/Compiler.jl | 14 +- Compiler/src/abstractinterpretation.jl | 4 +- Compiler/src/ssair/EscapeAnalysis.jl | 1 + Compiler/test/codegen.jl | 20 + base/Base_compiler.jl | 14 +- base/errorshow.jl | 2 +- base/invalidation.jl | 8 +- base/iterators.jl | 1 + base/runtime_internals.jl | 30 +- base/show.jl | 19 +- base/sysimg.jl | 2 - src/jl_exported_funcs.inc | 1 - src/julia.h | 35 +- src/julia_internal.h | 29 +- src/method.c | 22 +- src/module.c | 554 +++++++++++++++---------- src/staticdata.c | 6 +- src/toplevel.c | 6 +- test/core.jl | 52 ++- test/reflection.jl | 21 + 20 files changed, 533 insertions(+), 308 deletions(-) diff --git a/Compiler/src/Compiler.jl b/Compiler/src/Compiler.jl index f28bc02301dfd..981001cb2fbe6 100644 --- a/Compiler/src/Compiler.jl +++ b/Compiler/src/Compiler.jl @@ -35,10 +35,6 @@ else @eval baremodule Compiler -# Needs to match UUID defined in Project.toml -ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Compiler, - (0x807dbc54_b67e_4c79, 0x8afb_eafe4df6f2e1)) - using Core.Intrinsics, Core.IR using Core: ABIOverride, Builtin, CodeInstance, IntrinsicFunction, MethodInstance, MethodMatch, @@ -61,7 +57,7 @@ using Base: @_foldable_meta, @_gc_preserve_begin, @_gc_preserve_end, @nospeciali generating_output, get_nospecializeinfer_sig, get_world_counter, has_free_typevars, hasgenerator, hasintersect, indexed_iterate, isType, is_file_tracked, is_function_def, is_meta_expr, is_meta_expr_head, is_nospecialized, is_nospecializeinfer, is_defined_const_binding, - is_some_const_binding, is_some_guard, is_some_imported, is_valid_intrinsic_elptr, + is_some_const_binding, is_some_guard, is_some_imported, is_some_explicit_imported, is_some_binding_imported, is_valid_intrinsic_elptr, isbitsunion, isconcretedispatch, isdispatchelem, isexpr, isfieldatomic, isidentityfree, iskindtype, ismutabletypename, ismutationfree, issingletontype, isvarargtype, isvatuple, kwerr, lookup_binding_partition, may_invoke_generator, methods, midpoint, moduleroot, @@ -75,6 +71,10 @@ import Base: ==, _topmod, append!, convert, copy, copy!, findall, first, get, ge getindex, haskey, in, isempty, isready, iterate, iterate, last, length, max_world, min_world, popfirst!, push!, resize!, setindex!, size, intersect +# Needs to match UUID defined in Project.toml +ccall(:jl_set_module_uuid, Cvoid, (Any, NTuple{2, UInt64}), Compiler, + (0x807dbc54_b67e_4c79, 0x8afb_eafe4df6f2e1)) + const getproperty = Core.getfield const setproperty! = Core.setfield! const swapproperty! = Core.swapfield! @@ -130,7 +130,7 @@ something(x::Any, y...) = x ############ baremodule BuildSettings -using Core: ARGS, include +using Core: ARGS, include, Int, === using ..Compiler: >, getindex, length global MAX_METHODS::Int = 3 @@ -190,7 +190,7 @@ macro __SOURCE_FILE__() end module IRShow end # relies on string and IO operations defined in Base -baremodule TrimVerifier end # relies on IRShow, so define this afterwards +baremodule TrimVerifier using Core end # relies on IRShow, so define this afterwards if isdefined(Base, :end_base_include) # When this module is loaded as the standard library, include these files as usual diff --git a/Compiler/src/abstractinterpretation.jl b/Compiler/src/abstractinterpretation.jl index c0b079d700bee..f394542877e12 100644 --- a/Compiler/src/abstractinterpretation.jl +++ b/Compiler/src/abstractinterpretation.jl @@ -3242,7 +3242,7 @@ function abstract_eval_isdefinedglobal(interp::AbstractInterpreter, mod::Module, if allow_import !== true gr = GlobalRef(mod, sym) partition = lookup_binding_partition!(interp, gr, sv) - if allow_import !== true && is_some_imported(binding_kind(partition)) + if allow_import !== true && is_some_binding_imported(binding_kind(partition)) if allow_import === false rt = Const(false) else @@ -3495,7 +3495,7 @@ end function walk_binding_partition(imported_binding::Core.Binding, partition::Core.BindingPartition, world::UInt) valid_worlds = WorldRange(partition.min_world, partition.max_world) - while is_some_imported(binding_kind(partition)) + while is_some_binding_imported(binding_kind(partition)) imported_binding = partition_restriction(partition)::Core.Binding partition = lookup_binding_partition(world, imported_binding) valid_worlds = intersect(valid_worlds, WorldRange(partition.min_world, partition.max_world)) diff --git a/Compiler/src/ssair/EscapeAnalysis.jl b/Compiler/src/ssair/EscapeAnalysis.jl index af8e9b1a4959e..4ce972937700c 100644 --- a/Compiler/src/ssair/EscapeAnalysis.jl +++ b/Compiler/src/ssair/EscapeAnalysis.jl @@ -15,6 +15,7 @@ using Base: Base # imports import Base: ==, copy, getindex, setindex! # usings +using Core using Core: Builtin, IntrinsicFunction, SimpleVector, ifelse, sizeof using Core.IR using Base: # Base definitions diff --git a/Compiler/test/codegen.jl b/Compiler/test/codegen.jl index 4514852a2c0bc..4e0a1b1f88997 100644 --- a/Compiler/test/codegen.jl +++ b/Compiler/test/codegen.jl @@ -1007,3 +1007,23 @@ let end nothing end + +# Test that turning an implicit import into an explicit one doesn't pessimize codegen +module TurnedIntoExplicit + using Test + import ..get_llvm + + module ReExportBitCast + export bitcast + import Base: bitcast + end + using .ReExportBitCast + + f(x::UInt) = bitcast(Float64, x) + + @test !occursin("jl_apply_generic", get_llvm(f, Tuple{UInt})) + + import Base: bitcast + + @test !occursin("jl_apply_generic", get_llvm(f, Tuple{UInt})) +end diff --git a/base/Base_compiler.jl b/base/Base_compiler.jl index 640b8226788cb..911659034a145 100644 --- a/base/Base_compiler.jl +++ b/base/Base_compiler.jl @@ -277,18 +277,21 @@ include("operators.jl") include("pointer.jl") include("refvalue.jl") include("cmem.jl") + +function nextfloat end +function prevfloat end include("rounding.jl") include("float.jl") -include("checked.jl") -using .Checked -function cld end -function fld end - # Lazy strings import Core: String include("strings/lazy.jl") +function cld end +function fld end +include("checked.jl") +using .Checked + # array structures include("indices.jl") include("genericmemory.jl") @@ -373,3 +376,4 @@ Core._setparser!(fl_parse) # Further definition of Base will happen in Base.jl if loaded. end # baremodule Base +using .Base diff --git a/base/errorshow.jl b/base/errorshow.jl index 0fb6d0782e85f..16634efe97d8a 100644 --- a/base/errorshow.jl +++ b/base/errorshow.jl @@ -1157,7 +1157,7 @@ function UndefVarError_hint(io::IO, ex::UndefVarError) "with the module it should come from.") elseif kind === Base.PARTITION_KIND_GUARD print(io, "\nSuggestion: check for spelling errors or missing imports.") - elseif Base.is_some_imported(kind) + elseif Base.is_some_explicit_imported(kind) print(io, "\nSuggestion: this global was defined as `$(Base.partition_restriction(bpart).globalref)` but not assigned a value.") end elseif scope === :static_parameter diff --git a/base/invalidation.jl b/base/invalidation.jl index 5b48af0171205..14b88e71b9def 100644 --- a/base/invalidation.jl +++ b/base/invalidation.jl @@ -151,7 +151,9 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core latest_bpart = edge.partitions latest_bpart.max_world == typemax(UInt) || continue is_some_imported(binding_kind(latest_bpart)) || continue - partition_restriction(latest_bpart) === b || continue + if is_some_binding_imported(binding_kind(latest_bpart)) + partition_restriction(latest_bpart) === b || continue + end invalidate_code_for_globalref!(edge, latest_bpart, latest_bpart, new_max_world) else invalidate_method_for_globalref!(gr, edge::Method, invalidated_bpart, new_max_world) @@ -171,9 +173,9 @@ function invalidate_code_for_globalref!(b::Core.Binding, invalidated_bpart::Core isdefined(user_binding, :partitions) || continue latest_bpart = user_binding.partitions latest_bpart.max_world == typemax(UInt) || continue - binding_kind(latest_bpart) in (PARTITION_KIND_IMPLICIT, PARTITION_KIND_FAILED, PARTITION_KIND_GUARD) || continue + is_some_implicit(binding_kind(latest_bpart)) || continue new_bpart = need_to_invalidate_export ? - ccall(:jl_maybe_reresolve_implicit, Any, (Any, Any, Csize_t), user_binding, latest_bpart, new_max_world) : + ccall(:jl_maybe_reresolve_implicit, Any, (Any, Csize_t), user_binding, new_max_world) : latest_bpart if need_to_invalidate_code || new_bpart !== latest_bpart invalidate_code_for_globalref!(convert(Core.Binding, user_binding), latest_bpart, new_bpart, new_max_world) diff --git a/base/iterators.jl b/base/iterators.jl index d6367ed8d996e..c7450781c4928 100644 --- a/base/iterators.jl +++ b/base/iterators.jl @@ -17,6 +17,7 @@ using .Base: any, _counttuple, eachindex, ntuple, zero, prod, reduce, in, firstindex, lastindex, tail, fieldtypes, min, max, minimum, zero, oneunit, promote, promote_shape, LazyString, afoldl +using Core using Core: @doc using .Base: diff --git a/base/runtime_internals.jl b/base/runtime_internals.jl index 0bfb00b17a630..510d27ecb3701 100644 --- a/base/runtime_internals.jl +++ b/base/runtime_internals.jl @@ -197,17 +197,18 @@ function _fieldnames(@nospecialize t) end # N.B.: Needs to be synced with julia.h -const PARTITION_KIND_CONST = 0x0 -const PARTITION_KIND_CONST_IMPORT = 0x1 -const PARTITION_KIND_GLOBAL = 0x2 -const PARTITION_KIND_IMPLICIT = 0x3 -const PARTITION_KIND_EXPLICIT = 0x4 -const PARTITION_KIND_IMPORTED = 0x5 -const PARTITION_KIND_FAILED = 0x6 -const PARTITION_KIND_DECLARED = 0x7 -const PARTITION_KIND_GUARD = 0x8 -const PARTITION_KIND_UNDEF_CONST = 0x9 -const PARTITION_KIND_BACKDATED_CONST = 0xa +const PARTITION_KIND_CONST = 0x0 +const PARTITION_KIND_CONST_IMPORT = 0x1 +const PARTITION_KIND_GLOBAL = 0x2 +const PARTITION_KIND_IMPLICIT_GLOBAL = 0x3 +const PARTITION_KIND_IMPLICIT_CONST = 0x4 +const PARTITION_KIND_EXPLICIT = 0x5 +const PARTITION_KIND_IMPORTED = 0x6 +const PARTITION_KIND_FAILED = 0x7 +const PARTITION_KIND_DECLARED = 0x8 +const PARTITION_KIND_GUARD = 0x9 +const PARTITION_KIND_UNDEF_CONST = 0xa +const PARTITION_KIND_BACKDATED_CONST = 0xb const PARTITION_FLAG_EXPORTED = 0x10 const PARTITION_FLAG_DEPRECATED = 0x20 @@ -218,9 +219,12 @@ const PARTITION_MASK_FLAG = 0xf0 const BINDING_FLAG_ANY_IMPLICIT_EDGES = 0x8 -is_defined_const_binding(kind::UInt8) = (kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_BACKDATED_CONST) +is_defined_const_binding(kind::UInt8) = (kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_BACKDATED_CONST) is_some_const_binding(kind::UInt8) = (is_defined_const_binding(kind) || kind == PARTITION_KIND_UNDEF_CONST) -is_some_imported(kind::UInt8) = (kind == PARTITION_KIND_IMPLICIT || kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) +is_some_imported(kind::UInt8) = (kind == PARTITION_KIND_IMPLICIT_GLOBAL || kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) +is_some_implicit(kind::UInt8) = (kind == PARTITION_KIND_IMPLICIT_GLOBAL || kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_GUARD || kind == PARTITION_KIND_FAILED) +is_some_explicit_imported(kind::UInt8) = (kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED) +is_some_binding_imported(kind::UInt8) = is_some_explicit_imported(kind) || kind == PARTITION_KIND_IMPLICIT_GLOBAL is_some_guard(kind::UInt8) = (kind == PARTITION_KIND_GUARD || kind == PARTITION_KIND_FAILED || kind == PARTITION_KIND_UNDEF_CONST) function lookup_binding_partition(world::UInt, b::Core.Binding) diff --git a/base/show.jl b/base/show.jl index 055f0923d9432..48dff23a8af0a 100644 --- a/base/show.jl +++ b/base/show.jl @@ -1023,14 +1023,20 @@ end function isvisible(sym::Symbol, parent::Module, from::Module) isdeprecated(parent, sym) && return false isdefinedglobal(from, sym) || return false + isdefinedglobal(parent, sym) || return false parent_binding = convert(Core.Binding, GlobalRef(parent, sym)) from_binding = convert(Core.Binding, GlobalRef(from, sym)) while true from_binding === parent_binding && return true partition = lookup_binding_partition(tls_world_age(), from_binding) - is_some_imported(binding_kind(partition)) || break + is_some_explicit_imported(binding_kind(partition)) || break from_binding = partition_restriction(partition)::Core.Binding end + parent_partition = lookup_binding_partition(tls_world_age(), parent_binding) + from_partition = lookup_binding_partition(tls_world_age(), from_binding) + if is_defined_const_binding(binding_kind(parent_partition)) && is_defined_const_binding(binding_kind(from_partition)) + return parent_partition.restriction === from_partition.restriction + end return false end @@ -3391,7 +3397,7 @@ function print_partition(io::IO, partition::Core.BindingPartition) if kind == PARTITION_KIND_BACKDATED_CONST print(io, "backdated constant binding to ") print(io, partition_restriction(partition)) - elseif is_defined_const_binding(kind) + elseif kind == PARTITION_KIND_CONST print(io, "constant binding to ") print(io, partition_restriction(partition)) elseif kind == PARTITION_KIND_UNDEF_CONST @@ -3402,9 +3408,12 @@ function print_partition(io::IO, partition::Core.BindingPartition) print(io, "ambiguous binding - guard entry") elseif kind == PARTITION_KIND_DECLARED print(io, "weak global binding declared using `global` (implicit type Any)") - elseif kind == PARTITION_KIND_IMPLICIT - print(io, "implicit `using` from ") + elseif kind == PARTITION_KIND_IMPLICIT_GLOBAL + print(io, "implicit `using` resolved to global ") print(io, partition_restriction(partition).globalref) + elseif kind == PARTITION_KIND_IMPLICIT_CONST + print(io, "implicit `using` resolved to constant ") + print(io, partition_restriction(partition)) elseif kind == PARTITION_KIND_EXPLICIT print(io, "explicit `using` from ") print(io, partition_restriction(partition).globalref) @@ -3427,7 +3436,7 @@ function show(io::IO, ::MIME"text/plain", bnd::Core.Binding) print(io, "Binding ") print(io, bnd.globalref) if !isdefined(bnd, :partitions) - print(io, "No partitions") + print(io, " - No partitions") else partition = @atomic bnd.partitions while true diff --git a/base/sysimg.jl b/base/sysimg.jl index 42f54a849f157..c12ddcd71c66f 100644 --- a/base/sysimg.jl +++ b/base/sysimg.jl @@ -12,8 +12,6 @@ Core.include(Base, "Base.jl") had_compiler && ccall(:jl_init_restored_module, Cvoid, (Any,), Base) end -using .Base - # Set up Main module using Base.MainInclude # ans, err, and sometimes Out diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 06c563dcc0fa5..df1bdac23abe8 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -192,7 +192,6 @@ XX(jl_get_ARCH) \ XX(jl_get_backtrace) \ XX(jl_get_binding) \ - XX(jl_get_binding_for_method_def) \ XX(jl_get_binding_wr) \ XX(jl_check_binding_currently_writable) \ XX(jl_get_cpu_name) \ diff --git a/src/julia.h b/src/julia.h index 694e16ca31a97..30e4eb71f2320 100644 --- a/src/julia.h +++ b/src/julia.h @@ -632,7 +632,8 @@ typedef struct _jl_weakref_t { // These binding kinds depend solely on the set of using'd packages and are not explicitly // declared: // -// PARTITION_KIND_IMPLICIT +// PARTITION_KIND_IMPLICIT_CONST +// PARTITION_KIND_IMPLICIT_GLOBAL // PARTITION_KIND_GUARD // PARTITION_KIND_FAILED // @@ -683,37 +684,41 @@ enum jl_partition_kind { // `global x::T` to implicitly through a syntactic global assignment. // -> restriction holds the type restriction PARTITION_KIND_GLOBAL = 0x2, - // Implicit: The binding was implicitly imported from a `using`'d module. - // ->restriction holds the imported binding - PARTITION_KIND_IMPLICIT = 0x3, + // Implicit: The binding was a global, implicitly imported from a `using`'d module. + // ->restriction holds the ultimately imported global binding + PARTITION_KIND_IMPLICIT_GLOBAL = 0x3, + // Implicit: The binding was a constant, implicitly imported from a `using`'d module. + // ->restriction holds the ultimately imported constant value + PARTITION_KIND_IMPLICIT_CONST = 0x4, // Explicit: The binding was explicitly `using`'d by name // ->restriction holds the imported binding - PARTITION_KIND_EXPLICIT = 0x4, + PARTITION_KIND_EXPLICIT = 0x5, // Imported: The binding was explicitly `import`'d by name // ->restriction holds the imported binding - PARTITION_KIND_IMPORTED = 0x5, + PARTITION_KIND_IMPORTED = 0x6, // Failed: We attempted to import the binding, but the import was ambiguous // ->restriction is NULL. - PARTITION_KIND_FAILED = 0x6, + PARTITION_KIND_FAILED = 0x7, // Declared: The binding was declared using `global` or similar. This acts in most ways like // PARTITION_KIND_GLOBAL with an `Any` restriction, except that it may be redefined to a stronger // binding like `const` or an explicit import. // ->restriction is NULL. - PARTITION_KIND_DECLARED = 0x7, + PARTITION_KIND_DECLARED = 0x8, // Guard: The binding was looked at, but no global or import was resolved at the time // ->restriction is NULL. - PARTITION_KIND_GUARD = 0x8, + PARTITION_KIND_GUARD = 0x9, // Undef Constant: This binding partition is a constant declared using `const`, but // without a value. // ->restriction is NULL - PARTITION_KIND_UNDEF_CONST = 0x9, + PARTITION_KIND_UNDEF_CONST = 0xa, // Backated constant. A constant that was backdated for compatibility. In all other // ways equivalent to PARTITION_KIND_CONST, but prints a warning on access - PARTITION_KIND_BACKDATED_CONST = 0xa, + PARTITION_KIND_BACKDATED_CONST = 0xb, // This is not a real binding kind, but can be used to ask for a re-resolution // of the implicit binding kind - PARTITION_KIND_IMPLICIT_RECOMPUTE = 0xb + PARTITION_FAKE_KIND_IMPLICIT_RECOMPUTE = 0xc, + PARTITION_FAKE_KIND_CYCLE = 0xd }; static const uint8_t PARTITION_MASK_KIND = 0x0f; @@ -745,9 +750,9 @@ typedef struct JL_ALIGNED_ATTR(8) _jl_binding_partition_t { /* union { * // For ->kind == PARTITION_KIND_GLOBAL * jl_value_t *type_restriction; - * // For ->kind == PARTITION_KIND_CONST(_IMPORT) + * // For ->kind in (PARTITION_KIND_CONST(_IMPORT), PARTITION_KIND_IMPLICIT_CONST) * jl_value_t *constval; - * // For ->kind in (PARTITION_KIND_IMPLICIT, PARTITION_KIND_EXPLICIT, PARTITION_KIND_IMPORT) + * // For ->kind in (PARTITION_KIND_IMPLICIT_GLOBAL, PARTITION_KIND_EXPLICIT, PARTITION_KIND_IMPORT) * jl_binding_t *imported; * } restriction; */ @@ -2099,7 +2104,7 @@ JL_DLLEXPORT jl_value_t *jl_get_binding_type(jl_module_t *m, jl_sym_t *var); // get binding for assignment JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module_t *m, jl_sym_t *s); JL_DLLEXPORT jl_binding_t *jl_get_binding_wr(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var); -JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *var, size_t new_world); +JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b JL_PROPAGATES_ROOT, size_t new_world); JL_DLLEXPORT int jl_boundp(jl_module_t *m, jl_sym_t *var, int allow_import); JL_DLLEXPORT int jl_defines_or_exports_p(jl_module_t *m, jl_sym_t *var); JL_DLLEXPORT int jl_is_const(jl_module_t *m, jl_sym_t *var); diff --git a/src/julia_internal.h b/src/julia_internal.h index 8d02cce9a80d9..479ccbf961e71 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -855,7 +855,6 @@ typedef struct _modstack_t { jl_binding_t *b; struct _modstack_t *prev; } modstack_t; -void jl_check_new_binding_implicit(jl_binding_partition_t *new_bpart, jl_binding_t *b, modstack_t *st, size_t world); #ifndef __clang_gcanalyzer__ // The analyzer doesn't like looking through the arraylist, so just model the @@ -918,6 +917,7 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked(jl_binding_t *b J jl_binding_partition_t *old_bpart, jl_value_t *restriction_val, enum jl_partition_kind kind, size_t new_world) JL_GLOBALLY_ROOTED; JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b JL_PROPAGATES_ROOT, jl_binding_partition_t *old_bpart, jl_value_t *restriction_val, size_t kind, size_t new_world) JL_GLOBALLY_ROOTED; +JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t *bpart); extern jl_array_t *jl_module_init_order JL_GLOBALLY_ROOTED; extern htable_t jl_current_modules JL_GLOBALLY_ROOTED; extern JL_DLLEXPORT jl_module_t *jl_precompile_toplevel_module JL_GLOBALLY_ROOTED; @@ -937,7 +937,11 @@ jl_method_t *jl_make_opaque_closure_method(jl_module_t *module, jl_value_t *name JL_DLLEXPORT int jl_is_valid_oc_argtype(jl_tupletype_t *argt, jl_method_t *source); STATIC_INLINE int jl_bkind_is_some_import(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == PARTITION_KIND_IMPLICIT || kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED; + return kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_IMPLICIT_GLOBAL || kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED; +} + +STATIC_INLINE int jl_bkind_is_some_explicit_import(enum jl_partition_kind kind) JL_NOTSAFEPOINT { + return kind == PARTITION_KIND_EXPLICIT || kind == PARTITION_KIND_IMPORTED; } STATIC_INLINE int jl_bkind_is_some_guard(enum jl_partition_kind kind) JL_NOTSAFEPOINT { @@ -945,15 +949,15 @@ STATIC_INLINE int jl_bkind_is_some_guard(enum jl_partition_kind kind) JL_NOTSAFE } STATIC_INLINE int jl_bkind_is_some_implicit(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == PARTITION_KIND_IMPLICIT || jl_bkind_is_some_guard(kind); + return kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_IMPLICIT_GLOBAL || jl_bkind_is_some_guard(kind); } STATIC_INLINE int jl_bkind_is_some_constant(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_UNDEF_CONST || kind == PARTITION_KIND_BACKDATED_CONST; + return kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_UNDEF_CONST || kind == PARTITION_KIND_BACKDATED_CONST; } STATIC_INLINE int jl_bkind_is_defined_constant(enum jl_partition_kind kind) JL_NOTSAFEPOINT { - return kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_BACKDATED_CONST; + return kind == PARTITION_KIND_IMPLICIT_CONST || kind == PARTITION_KIND_CONST || kind == PARTITION_KIND_CONST_IMPORT || kind == PARTITION_KIND_BACKDATED_CONST; } JL_DLLEXPORT jl_binding_partition_t *jl_get_binding_partition(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world) JL_GLOBALLY_ROOTED; @@ -982,7 +986,8 @@ STATIC_INLINE void jl_walk_binding_inplace_worlds(jl_binding_t **bnd, jl_binding STATIC_INLINE void jl_walk_binding_inplace(jl_binding_t **bnd, jl_binding_partition_t **bpart, size_t world) JL_NOTSAFEPOINT { while (1) { - if (!jl_bkind_is_some_import(jl_binding_kind(*bpart))) + enum jl_partition_kind kind = jl_binding_kind(*bpart); + if (!jl_bkind_is_some_explicit_import(kind) && kind != PARTITION_KIND_IMPLICIT_GLOBAL) return; *bnd = (jl_binding_t*)(*bpart)->restriction; *bpart = jl_get_binding_partition(*bnd, world); @@ -994,14 +999,14 @@ STATIC_INLINE void jl_walk_binding_inplace_depwarn(jl_binding_t **bnd, jl_bindin int passed_explicit = 0; while (1) { enum jl_partition_kind kind = jl_binding_kind(*bpart); - if (!jl_bkind_is_some_import(kind)) { + if (!jl_bkind_is_some_explicit_import(kind) && kind != PARTITION_KIND_IMPLICIT_GLOBAL) { if (!passed_explicit && depwarn) *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; return; } if (!passed_explicit && depwarn) *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; - if (kind != PARTITION_KIND_IMPLICIT) + if (kind != PARTITION_KIND_IMPLICIT_GLOBAL) passed_explicit = 1; *bnd = (jl_binding_t*)(*bpart)->restriction; *bpart = jl_get_binding_partition(*bnd, world); @@ -1014,14 +1019,14 @@ STATIC_INLINE void jl_walk_binding_inplace_all(jl_binding_t **bnd, jl_binding_pa int passed_explicit = 0; while (*bpart) { enum jl_partition_kind kind = jl_binding_kind(*bpart); - if (!jl_bkind_is_some_import(kind)) { + if (!jl_bkind_is_some_explicit_import(kind) && kind != PARTITION_KIND_IMPLICIT_GLOBAL) { if (!passed_explicit && depwarn) *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; return; } if (!passed_explicit && depwarn) *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; - if (kind != PARTITION_KIND_IMPLICIT) + if (kind != PARTITION_KIND_IMPLICIT_GLOBAL) passed_explicit = 1; *bnd = (jl_binding_t*)(*bpart)->restriction; *bpart = jl_get_binding_partition_all(*bnd, min_world, max_world); @@ -1038,14 +1043,14 @@ STATIC_INLINE void jl_walk_binding_inplace_worlds(jl_binding_t **bnd, jl_binding if (*max_world > bpart_max_world) *max_world = bpart_max_world; enum jl_partition_kind kind = jl_binding_kind(*bpart); - if (!jl_bkind_is_some_import(kind)) { + if (!jl_bkind_is_some_explicit_import(kind) && kind != PARTITION_KIND_IMPLICIT_GLOBAL) { if (!passed_explicit && depwarn) *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; return; } if (!passed_explicit && depwarn) *depwarn |= (*bpart)->kind & PARTITION_FLAG_DEPWARN; - if (kind != PARTITION_KIND_IMPLICIT) + if (kind != PARTITION_KIND_IMPLICIT_GLOBAL) passed_explicit = 1; *bnd = (jl_binding_t*)(*bpart)->restriction; *bpart = jl_get_binding_partition(*bnd, world); diff --git a/src/method.c b/src/method.c index 11609f9d3b61d..353fbdaf0ecff 100644 --- a/src/method.c +++ b/src/method.c @@ -1102,26 +1102,18 @@ JL_DLLEXPORT jl_value_t *jl_declare_const_gf(jl_module_t *mod, jl_sym_t *name) { JL_LOCK(&world_counter_lock); size_t new_world = jl_atomic_load_relaxed(&jl_world_counter) + 1; - jl_binding_t *b = jl_get_binding_for_method_def(mod, name, new_world); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); - jl_value_t *gf = NULL; - enum jl_partition_kind kind = jl_binding_kind(bpart); - if (!jl_bkind_is_some_guard(kind) && kind != PARTITION_KIND_DECLARED && kind != PARTITION_KIND_IMPLICIT) { - jl_walk_binding_inplace(&b, &bpart, new_world); - if (jl_bkind_is_some_constant(jl_binding_kind(bpart))) { - gf = bpart->restriction; - JL_GC_PROMISE_ROOTED(gf); - jl_check_gf(gf, b->globalref->name); - JL_UNLOCK(&world_counter_lock); - return gf; - } - jl_errorf("cannot define function %s; it already has a value", jl_symbol_name(name)); + jl_binding_t *b = jl_get_module_binding(mod, name, 1); + jl_value_t *gf = jl_get_existing_strong_gf(b, new_world); + if (gf) { + jl_check_gf(gf, name); + JL_UNLOCK(&world_counter_lock); + return gf; } gf = (jl_value_t*)jl_new_generic_function(name, mod, new_world); // From this point on (if we didn't error), we're committed to raising the world age, // because we've used it to declare the type name. - jl_atomic_store_release(&jl_world_counter, new_world); jl_declare_constant_val3(b, mod, name, gf, PARTITION_KIND_CONST, new_world); + jl_atomic_store_release(&jl_world_counter, new_world); JL_GC_PROMISE_ROOTED(gf); JL_UNLOCK(&world_counter_lock); return gf; diff --git a/src/module.c b/src/module.c index 1e138980a0dc0..1f20352b0ec30 100644 --- a/src/module.c +++ b/src/module.c @@ -26,54 +26,146 @@ static jl_binding_partition_t *new_binding_partition(void) return bpart; } +struct implicit_search_gap { + _Atomic(jl_binding_partition_t *) *insert; + jl_binding_partition_t *replace; + jl_value_t *parent; + + size_t min_world; + size_t max_world; + size_t inherited_flags; +}; + +STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition__(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world, struct implicit_search_gap *gap) JL_GLOBALLY_ROOTED +{ + // Iterate through the list of binding partitions, keeping track of where to insert a new one for an implicit + // resolution if necessary. + while (gap->replace && world < gap->replace->min_world) { + gap->insert = &gap->replace->next; + gap->max_world = gap->replace->min_world - 1; + gap->parent = (jl_value_t*)gap->replace; + gap->replace = jl_atomic_load_relaxed(gap->insert); + } + if (gap->replace && world <= jl_atomic_load_relaxed(&gap->replace->max_world)) { + return gap->replace; + } + gap->min_world = gap->replace ? jl_atomic_load_relaxed(&gap->replace->max_world) + 1 : 0; + if (gap->replace) + gap->inherited_flags = gap->replace->kind & PARTITION_MASK_FLAG; + else + gap->inherited_flags = 0; + return NULL; +} -static jl_binding_partition_t *jl_get_binding_partition2(jl_binding_t *b, size_t world, modstack_t *st); +STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_if_present(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world, struct implicit_search_gap *gap) +{ + gap->parent = (jl_value_t*)b; + gap->insert = &b->partitions; + gap->replace = jl_atomic_load_relaxed(gap->insert); + gap->min_world = 0; + gap->max_world = ~(size_t)0; + gap->inherited_flags = 0; + return jl_get_binding_partition__(b, world, gap); +} + +struct implicit_search_resolution { + enum jl_partition_kind ultimate_kind; + jl_value_t *binding_or_const; + size_t min_world; + size_t max_world; + int saw_cycle; + //// Not semantic, but used for reflection. + // If non-null, the unique module from which this binding was imported + jl_module_t *debug_only_import_from; + // If non-null, the unique binding imported. For PARTITION_KIND_IMPLICIT_GLOBAL, always matches binding_or_const. + // Must have trust_cache = 0. + jl_binding_t *debug_only_ultimate_binding; +}; + +static size_t WORLDMAX(size_t a, size_t b) { return a > b ? a : b; } +static size_t WORLDMIN(size_t a, size_t b) { return a > b ? b : a; } + +static void update_implicit_resolution(struct implicit_search_resolution *to_update, struct implicit_search_resolution resolution) +{ + to_update->min_world = WORLDMAX(to_update->min_world, resolution.min_world); + to_update->max_world = WORLDMIN(to_update->max_world, resolution.max_world); + to_update->saw_cycle |= resolution.saw_cycle; + if (resolution.ultimate_kind == PARTITION_FAKE_KIND_CYCLE) { + // Cycles get ignored. This causes the resolution resolution to only be partial, so we can't + // cache it. This gets tracked in saw_cycle; + to_update->saw_cycle = 1; + return; + } + if (resolution.ultimate_kind == PARTITION_KIND_GUARD) { + // Ignore guard imports + return; + } + if (to_update->ultimate_kind == PARTITION_KIND_GUARD) { + assert(resolution.binding_or_const); + to_update->ultimate_kind = resolution.ultimate_kind; + to_update->binding_or_const = resolution.binding_or_const; + to_update->debug_only_import_from = resolution.debug_only_import_from; + to_update->debug_only_ultimate_binding = resolution.debug_only_ultimate_binding; + return; + } + if (resolution.ultimate_kind == to_update->ultimate_kind && + resolution.binding_or_const == to_update->binding_or_const) { + if (resolution.debug_only_import_from != to_update->debug_only_import_from) { + to_update->debug_only_import_from = NULL; + } + if (resolution.debug_only_ultimate_binding != to_update->debug_only_ultimate_binding) { + to_update->debug_only_ultimate_binding = NULL; + } + return; + } + to_update->ultimate_kind = PARTITION_KIND_FAILED; + to_update->binding_or_const = NULL; + to_update->debug_only_import_from = NULL; + to_update->debug_only_ultimate_binding = NULL; +} -static int eq_bindings(jl_binding_partition_t *owner, jl_binding_t *alias, size_t world) +static jl_binding_partition_t *jl_implicit_import_resolved(jl_binding_t *b, struct implicit_search_gap gap, struct implicit_search_resolution resolution) { - jl_binding_t *ownerb = NULL; - jl_binding_partition_t *alias_bpart = jl_get_binding_partition(alias, world); - if (owner == alias_bpart) - return 1; - jl_walk_binding_inplace(&ownerb, &owner, world); - jl_walk_binding_inplace(&alias, &alias_bpart, world); - if (jl_bkind_is_some_constant(jl_binding_kind(owner)) && - jl_bkind_is_some_constant(jl_binding_kind(alias_bpart)) && - owner->restriction && - alias_bpart->restriction == owner->restriction) - return 1; - return owner == alias_bpart; + jl_binding_partition_t *new_bpart = new_binding_partition(); + jl_atomic_store_relaxed(&new_bpart->max_world, gap.max_world < resolution.max_world ? gap.max_world : resolution.max_world); + new_bpart->min_world = gap.min_world > resolution.min_world ? gap.min_world : resolution.min_world; + new_bpart->kind = resolution.ultimate_kind | gap.inherited_flags; + new_bpart->restriction = resolution.binding_or_const; + jl_gc_wb_fresh(new_bpart, new_bpart->restriction); + jl_atomic_store_relaxed(&new_bpart->next, gap.replace); + if (!jl_atomic_cmpswap(gap.insert, &gap.replace, new_bpart)) + return NULL; + jl_gc_wb(gap.parent, new_bpart); + return new_bpart; } // find a binding from a module's `usings` list -void jl_check_new_binding_implicit( - jl_binding_partition_t *new_bpart, jl_binding_t *b, modstack_t *st, size_t world) -{ - modstack_t top = { b, st }; - modstack_t *tmp = st; - for (; tmp != NULL; tmp = tmp->prev) { - if (tmp->b == b) { - new_bpart->restriction = NULL; - new_bpart->kind = PARTITION_KIND_FAILED; /* PARTITION_KIND_CYCLE */ - return; +struct implicit_search_resolution jl_resolve_implicit_import(jl_binding_t *b, modstack_t *st, size_t world, int trust_cache) +{ + // First check if we've hit a cycle in this resolution + { + modstack_t *tmp = st; + for (; tmp != NULL; tmp = tmp->prev) { + if (tmp->b == b) { + return (struct implicit_search_resolution){ PARTITION_FAKE_KIND_CYCLE, NULL, 0, ~(size_t)0, 1, NULL, NULL }; + } } } jl_module_t *m = b->globalref->mod; jl_sym_t *var = b->globalref->name; - jl_binding_t *deprecated_impb = NULL; - jl_binding_t *impb = NULL; - jl_binding_partition_t *impbpart = NULL; - - size_t min_world = new_bpart->min_world; - size_t max_world = jl_atomic_load_relaxed(&new_bpart->max_world); + modstack_t top = { b, st }; + struct implicit_search_resolution impstate; + struct implicit_search_resolution depimpstate; + size_t min_world = 0; + size_t max_world = ~(size_t)0; + impstate = depimpstate = (struct implicit_search_resolution){ PARTITION_KIND_GUARD, NULL, min_world, max_world, 0, NULL, NULL }; JL_LOCK(&m->lock); int i = (int)module_usings_length(m) - 1; JL_UNLOCK(&m->lock); - enum jl_partition_kind guard_kind = PARTITION_KIND_GUARD; - for (; i >= 0; --i) { + for (; i >= 0 && impstate.ultimate_kind != PARTITION_KIND_FAILED; --i) { JL_LOCK(&m->lock); struct _jl_module_using data = *module_usings_getidx(m, i); JL_UNLOCK(&m->lock); @@ -87,144 +179,150 @@ void jl_check_new_binding_implicit( min_world = data.max_world + 1; continue; } + + min_world = WORLDMAX(min_world, data.min_world); + max_world = WORLDMIN(max_world, data.max_world); + jl_module_t *imp = data.mod; JL_GC_PROMISE_ROOTED(imp); jl_binding_t *tempb = jl_get_module_binding(imp, var, 0); - if (tempb != NULL) { - if (data.min_world > min_world) - min_world = data.min_world; - if (data.max_world < min_world) - max_world = data.max_world; - - jl_binding_partition_t *tempbpart = jl_get_binding_partition2(tempb, world, &top); - JL_GC_PROMISE_ROOTED(tempbpart); - - size_t tempbmax_world = jl_atomic_load_relaxed(&tempbpart->max_world); - if (tempbpart->min_world > min_world) - min_world = tempbpart->min_world; - if (tempbmax_world < max_world) - max_world = tempbmax_world; - - // N.B.: Which aspects of the partition are considered here needs to - // be kept in sync with `export_affecting_partition_flags` in the - // invalidation code. - if ((tempbpart->kind & PARTITION_FLAG_EXPORTED) == 0) - continue; + if (!tempb) { + // If the binding has never been allocated, it could not have been marked exported, so + // it is irrelevant for our resolution. We can move on. + continue; + } - if (impb) { - if (tempbpart->kind & PARTITION_FLAG_DEPRECATED) - continue; - if (jl_binding_kind(tempbpart) == PARTITION_KIND_GUARD && - jl_binding_kind(impbpart) != PARTITION_KIND_GUARD) - continue; - if (jl_binding_kind(impbpart) == PARTITION_KIND_GUARD) { - impb = tempb; - impbpart = tempbpart; - continue; - } - if (eq_bindings(tempbpart, impb, world)) - continue; - // Binding is ambiguous - // TODO: Even for eq bindings, this may need to further constrain the world age. - deprecated_impb = impb = NULL; - guard_kind = PARTITION_KIND_FAILED; - break; + struct implicit_search_gap gap; + jl_binding_partition_t *tempbpart = jl_get_binding_partition_if_present(tempb, world, &gap); + size_t tempbpart_flags = tempbpart ? (tempbpart->kind & PARTITION_MASK_FLAG) : gap.inherited_flags; + + while (tempbpart && jl_bkind_is_some_explicit_import(jl_binding_kind(tempbpart))) { + max_world = WORLDMIN(max_world, jl_atomic_load_relaxed(&tempbpart->max_world)); + min_world = WORLDMAX(min_world, tempbpart->min_world); + + tempb = (jl_binding_t*)tempbpart->restriction; + tempbpart = jl_get_binding_partition_if_present(tempb, world, &gap); + } + + int tempbpart_valid = tempbpart && (trust_cache || !jl_bkind_is_some_implicit(jl_binding_kind(tempbpart))); + size_t tembppart_max_world = tempbpart_valid ? jl_atomic_load_relaxed(&tempbpart->max_world) : gap.max_world; + size_t tembppart_min_world = tempbpart ? WORLDMAX(tempbpart->min_world, gap.min_world) : gap.min_world; + + max_world = WORLDMIN(max_world, tembppart_max_world); + min_world = WORLDMAX(min_world, tembppart_min_world); + + if (!(tempbpart_flags & PARTITION_FLAG_EXPORTED)) { + // Partition not exported - skip. + continue; + } + + struct implicit_search_resolution *comparison = &impstate; + if (impstate.ultimate_kind != PARTITION_KIND_GUARD) { + if (tempbpart_flags & PARTITION_FLAG_DEPRECATED) { + // Deprecated, but we already have a non-deprecated binding for this - skip. + continue; } - else if (tempbpart->kind & PARTITION_FLAG_DEPRECATED) { - if (deprecated_impb) { - if (!eq_bindings(tempbpart, deprecated_impb, world)) { - guard_kind = PARTITION_KIND_FAILED; - deprecated_impb = NULL; - } - } - else if (guard_kind == PARTITION_KIND_GUARD) { - deprecated_impb = tempb; - } + } else if (tempbpart_flags & PARTITION_FLAG_DEPRECATED) { + if (depimpstate.ultimate_kind == PARTITION_KIND_FAILED) { + // We've already decided that the deprecated bindings are ambiguous, so skip this, but + // keep going to look for non-deprecated bindings. + continue; } - else { - impb = tempb; - impbpart = tempbpart; + comparison = &depimpstate; + } + + struct implicit_search_resolution imp_resolution = { PARTITION_KIND_GUARD, NULL, min_world, max_world, 0, NULL, NULL }; + if (!tempbpart_valid) { + imp_resolution = jl_resolve_implicit_import(tempb, &top, world, trust_cache); + } else { + enum jl_partition_kind kind = jl_binding_kind(tempbpart); + if (kind == PARTITION_KIND_IMPLICIT_GLOBAL) { + imp_resolution.binding_or_const = tempbpart->restriction; + imp_resolution.debug_only_ultimate_binding = (jl_binding_t*)tempbpart->restriction; + imp_resolution.ultimate_kind = PARTITION_KIND_IMPLICIT_GLOBAL; + } else if (kind == PARTITION_KIND_GLOBAL || kind == PARTITION_KIND_DECLARED || kind == PARTITION_KIND_BACKDATED_CONST) { + imp_resolution.binding_or_const = (jl_value_t *)tempb; + imp_resolution.debug_only_ultimate_binding = tempb; + imp_resolution.ultimate_kind = PARTITION_KIND_IMPLICIT_GLOBAL; + } else if (jl_bkind_is_defined_constant(kind)) { + assert(tempbpart->restriction); + imp_resolution.binding_or_const = tempbpart->restriction; + imp_resolution.debug_only_ultimate_binding = tempb; + imp_resolution.ultimate_kind = PARTITION_KIND_IMPLICIT_CONST; } } - } + imp_resolution.debug_only_import_from = imp; + update_implicit_resolution(comparison, imp_resolution); - if (deprecated_impb && !impb) - impb = deprecated_impb; + if (!tempbpart && !imp_resolution.saw_cycle) { + // Independent of whether or not we trust the cache, we have independently computed the implicit resolution + // for this import, so we can put it in the cache. + jl_implicit_import_resolved(tempb, gap, imp_resolution); + } + } - assert(min_world <= max_world); - new_bpart->min_world = min_world; - jl_atomic_store_relaxed(&new_bpart->max_world, max_world); - if (impb) { - new_bpart->kind = PARTITION_KIND_IMPLICIT; - new_bpart->restriction = (jl_value_t*)impb; - jl_gc_wb(new_bpart, impb); - // TODO: World age constraints? - } else { - new_bpart->kind = guard_kind; - new_bpart->restriction = NULL; + if (impstate.ultimate_kind == PARTITION_KIND_GUARD && depimpstate.ultimate_kind != PARTITION_KIND_GUARD) { + depimpstate.min_world = WORLDMAX(depimpstate.min_world, min_world); + depimpstate.max_world = WORLDMIN(depimpstate.max_world, max_world); + return depimpstate; } + impstate.min_world = WORLDMAX(impstate.min_world, min_world); + impstate.max_world = WORLDMIN(impstate.max_world, max_world); + return impstate; } JL_DLLEXPORT jl_binding_partition_t *jl_maybe_reresolve_implicit(jl_binding_t *b, size_t new_max_world) { - jl_binding_partition_t *new_bpart = new_binding_partition(); - jl_binding_partition_t *bpart = jl_atomic_load_acquire(&b->partitions); - assert(bpart); - JL_GC_PUSH1(&new_bpart); + struct implicit_search_gap gap; while (1) { - jl_atomic_store_relaxed(&new_bpart->next, bpart); - jl_gc_wb(new_bpart, bpart); - jl_check_new_binding_implicit(new_bpart, b, NULL, new_max_world+1); - if (bpart->kind & PARTITION_FLAG_EXPORTED) - new_bpart->kind |= PARTITION_FLAG_EXPORTED; - if (new_bpart->kind == bpart->kind && new_bpart->restriction == bpart->restriction) { - JL_GC_POP(); + jl_binding_partition_t *bpart = jl_get_binding_partition_if_present(b, new_max_world+1, &gap); + assert(bpart == jl_atomic_load_relaxed(&b->partitions)); + assert(bpart); + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, new_max_world+1, 0); + if (resolution.min_world == bpart->min_world) { + assert(bpart->restriction == resolution.binding_or_const && jl_binding_kind(bpart) == resolution.ultimate_kind); return bpart; } - // Resolution changed, insert the new partition + assert(resolution.min_world == new_max_world+1 && "Missed an invalidation or bad resolution bounds"); size_t expected_max_world = ~(size_t)0; - if (jl_atomic_cmpswap(&bpart->max_world, &expected_max_world, new_max_world) && - jl_atomic_cmpswap(&b->partitions, &bpart, new_bpart)) { - jl_gc_wb(b, new_bpart); - JL_GC_POP(); - return new_bpart; + if (jl_atomic_cmpswap(&bpart->max_world, &expected_max_world, new_max_world)) + { + gap.min_world = new_max_world+1; + gap.inherited_flags = bpart->kind & PARTITION_MASK_FLAG; + jl_binding_partition_t *new_bpart = jl_implicit_import_resolved(b, gap, resolution); + if (new_bpart) + return new_bpart; } } } +JL_DLLEXPORT void jl_update_loaded_bpart(jl_binding_t *b, jl_binding_partition_t *bpart) +{ + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_atomic_load_relaxed(&jl_world_counter), 0); + bpart->min_world = resolution.min_world; + jl_atomic_store_relaxed(&bpart->max_world, resolution.max_world); + bpart->restriction = resolution.binding_or_const; + bpart->kind = resolution.ultimate_kind; +} + STATIC_INLINE jl_binding_partition_t *jl_get_binding_partition_(jl_binding_t *b JL_PROPAGATES_ROOT, jl_value_t *parent, _Atomic(jl_binding_partition_t *)*insert, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED { assert(jl_is_binding(b)); - jl_binding_partition_t *bpart = jl_atomic_load_relaxed(insert); - size_t max_world = (size_t)-1; - jl_binding_partition_t *new_bpart = NULL; - JL_GC_PUSH1(&new_bpart); + struct implicit_search_gap gap; + gap.parent = parent; + gap.insert = insert; + gap.inherited_flags = 0; + gap.min_world = 0; + gap.max_world = ~(size_t)0; while (1) { - while (bpart && world < bpart->min_world) { - insert = &bpart->next; - max_world = bpart->min_world - 1; - parent = (jl_value_t *)bpart; - bpart = jl_atomic_load_relaxed(&bpart->next); - } - if (bpart && world <= jl_atomic_load_relaxed(&bpart->max_world)) { - JL_GC_POP(); - return bpart; - } - if (!new_bpart) - new_bpart = new_binding_partition(); - jl_atomic_store_relaxed(&new_bpart->next, bpart); + gap.replace = jl_atomic_load_relaxed(gap.insert); + jl_binding_partition_t *bpart = jl_get_binding_partition__(b, world, &gap); if (bpart) - jl_gc_wb(new_bpart, bpart); // Not fresh the second time around the loop - new_bpart->min_world = bpart ? jl_atomic_load_relaxed(&bpart->max_world) + 1 : 0; - jl_atomic_store_relaxed(&new_bpart->max_world, max_world); - jl_check_new_binding_implicit(new_bpart, b, st, world); - if (bpart && (bpart->kind & PARTITION_FLAG_EXPORTED)) - new_bpart->kind |= PARTITION_FLAG_EXPORTED; - if (jl_atomic_cmpswap(insert, &bpart, new_bpart)) { - jl_gc_wb(parent, new_bpart); - JL_GC_POP(); + return bpart; + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, world, 1); + jl_binding_partition_t *new_bpart = jl_implicit_import_resolved(b, gap, resolution); + if (new_bpart) return new_bpart; - } } } @@ -242,11 +340,6 @@ jl_binding_partition_t *jl_get_binding_partition_with_hint(jl_binding_t *b, jl_b return jl_get_binding_partition_(b, (jl_value_t*)prev, &prev->next, world, NULL); } -jl_binding_partition_t *jl_get_binding_partition2(jl_binding_t *b JL_PROPAGATES_ROOT, size_t world, modstack_t *st) JL_GLOBALLY_ROOTED { - assert(b); - return jl_get_binding_partition_(b, (jl_value_t*)b, &b->partitions, world, st); -} - jl_binding_partition_t *jl_get_binding_partition_all(jl_binding_t *b, size_t min_world, size_t max_world) { if (!b) return NULL; @@ -275,14 +368,17 @@ JL_DLLEXPORT int jl_get_binding_leaf_partitions_restriction_kind(jl_binding_t *b size_t cur_min_world = bpart->min_world; size_t cur_max_world = validated_min_world - 1; jl_walk_binding_inplace_worlds(&curb, &curbpart, &cur_min_world, &cur_max_world, &maybe_depwarn, cur_max_world); + enum jl_partition_kind kind = jl_binding_kind(curbpart); + if (kind == PARTITION_KIND_IMPLICIT_CONST) + kind = PARTITION_KIND_CONST; if (first == 1) { - rkp->kind = jl_binding_kind(curbpart); + rkp->kind = kind; rkp->restriction = curbpart->restriction; if (rkp->kind == PARTITION_KIND_GLOBAL || rkp->kind == PARTITION_KIND_DECLARED) rkp->binding_if_global = curb; first = 0; } else { - if (jl_binding_kind(curbpart) != rkp->kind || curbpart->restriction != rkp->restriction) + if (kind != rkp->kind || curbpart->restriction != rkp->restriction) return 0; if ((rkp->kind == PARTITION_KIND_GLOBAL || rkp->kind == PARTITION_KIND_DECLARED) && rkp->binding_if_global != curb) return 0; @@ -381,18 +477,18 @@ JL_DLLEXPORT jl_binding_partition_t *jl_declare_constant_val3( jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); while (!new_bpart) { enum jl_partition_kind kind = jl_binding_kind(bpart); - if (jl_bkind_is_some_constant(kind)) { + if (jl_bkind_is_some_constant(kind) && !jl_bkind_is_some_implicit(kind)) { if (!val) { new_bpart = bpart; break; } jl_value_t *old = bpart->restriction; JL_GC_PROMISE_ROOTED(old); - if (jl_egal(val, old)) { + if (val == old || (val && old && jl_egal(val, old))) { new_bpart = bpart; break; } - } else if (jl_bkind_is_some_import(kind) && kind != PARTITION_KIND_IMPLICIT) { + } else if (jl_bkind_is_some_explicit_import(kind)) { jl_errorf("cannot declare %s.%s constant; it was already declared as an import", jl_symbol_name(mod->name), jl_symbol_name(var)); } else if (kind == PARTITION_KIND_GLOBAL) { @@ -619,7 +715,7 @@ extern void check_safe_newbinding(jl_module_t *m, jl_sym_t *var) } } -static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b, jl_module_t *m, jl_sym_t *var) JL_GLOBALLY_ROOTED; +static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b) JL_GLOBALLY_ROOTED; // Checks that the binding in general is currently writable, but does not perform any checks on the // value to be written into the binding. @@ -638,15 +734,15 @@ JL_DLLEXPORT void jl_check_binding_currently_writable(jl_binding_t *b, jl_module jl_symbol_name(m->name), jl_symbol_name(s), jl_symbol_name(s), jl_symbol_name(m->name)); } - else if (jl_bkind_is_some_constant(kind)) { + else if (jl_bkind_is_some_constant(kind) && kind != PARTITION_KIND_IMPLICIT_CONST) { jl_errorf("invalid assignment to constant %s.%s. This redefinition may be permitted using the `const` keyword.", jl_symbol_name(m->name), jl_symbol_name(s)); } else { - jl_module_t *from = jl_binding_dbgmodule(b, m, s); - if (from == m) + jl_module_t *from = jl_binding_dbgmodule(b); + if (from == m || !from) jl_errorf("cannot assign a value to imported variable %s.%s", - jl_symbol_name(from->name), jl_symbol_name(s)); + jl_symbol_name(m->name), jl_symbol_name(s)); else jl_errorf("cannot assign a value to imported variable %s.%s from module %s", jl_symbol_name(from->name), jl_symbol_name(s), jl_symbol_name(m->name)); @@ -667,6 +763,12 @@ JL_DLLEXPORT jl_module_t *jl_get_module_of_binding(jl_module_t *m, jl_sym_t *var jl_binding_t *b = jl_get_module_binding(m, var, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); jl_walk_binding_inplace(&b, &bpart, jl_current_task->world_age); + if (jl_binding_kind(bpart) == PARTITION_KIND_IMPLICIT_CONST) { + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_current_task->world_age, 0); + if (!resolution.debug_only_ultimate_binding) + jl_error("Constant binding was imported from multiple modules"); + b = resolution.debug_only_ultimate_binding; + } return b ? b->globalref->mod : m; } @@ -813,18 +915,37 @@ JL_DLLEXPORT jl_value_t *jl_bpart_get_restriction_value(jl_binding_partition_t * return v; } -// get binding for adding a method -// like jl_get_binding_wr, but has different error paths and messages -JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_t *var, size_t new_world) +// for error message printing: look up the module that exported a binding to m as var +// this might not be the same as the owner of the binding, since the binding itself may itself have been imported from elsewhere +static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b) +{ + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + enum jl_partition_kind kind = jl_binding_kind(bpart); + if (jl_bkind_is_some_explicit_import(kind) || kind == PARTITION_KIND_IMPLICIT_GLOBAL) { + return ((jl_binding_t*)bpart->restriction)->globalref->mod; + } + if (kind == PARTITION_KIND_IMPLICIT_CONST) { + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, jl_current_task->world_age, 1); + return resolution.debug_only_import_from; + } + return b->globalref->mod; +} + +// Look at the given binding and decide whether to add a new method to an existing generic function +// or ask for the creation of a new generic function (NULL return), checking various error conditions +// along the way. +JL_DLLEXPORT jl_value_t *jl_get_existing_strong_gf(jl_binding_t *b, size_t new_world) { - jl_binding_t *b = jl_get_module_binding(m, var, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, new_world); enum jl_partition_kind kind = jl_binding_kind(bpart); - if (kind == PARTITION_KIND_GLOBAL || kind == PARTITION_KIND_DECLARED || jl_bkind_is_some_constant(kind)) - return b; - if (jl_bkind_is_some_guard(kind)) { - check_safe_newbinding(m, var); - return b; + if (jl_bkind_is_some_constant(kind) && kind != PARTITION_KIND_IMPLICIT_CONST) + return bpart->restriction; + if (jl_bkind_is_some_guard(kind) || kind == PARTITION_KIND_DECLARED) { + check_safe_newbinding(b->globalref->mod, b->globalref->name); + return NULL; + } + if (!jl_bkind_is_some_import(kind)) { + jl_errorf("cannot define function %s; it already has a value", jl_symbol_name(b->globalref->name)); } jl_binding_t *ownerb = b; jl_walk_binding_inplace(&ownerb, &bpart, new_world); @@ -832,35 +953,36 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ if (jl_bkind_is_some_constant(jl_binding_kind(bpart))) f = bpart->restriction; if (f == NULL) { - if (kind == PARTITION_KIND_IMPLICIT) { - check_safe_newbinding(m, var); - return b; + if (jl_bkind_is_some_implicit(kind)) { + check_safe_newbinding(b->globalref->mod, b->globalref->name); + return NULL; } - jl_module_t *from = jl_binding_dbgmodule(b, m, var); - // we must have implicitly imported this with using, so call jl_binding_dbgmodule to try to get the name of the module we got this from + jl_module_t *from = jl_binding_dbgmodule(b);\ + assert(from); // Can only be NULL if implicit, which we excluded above jl_errorf("invalid method definition in %s: exported function %s.%s does not exist", - jl_module_debug_name(m), jl_module_debug_name(from), jl_symbol_name(var)); + jl_module_debug_name(b->globalref->mod), jl_module_debug_name(from), jl_symbol_name(b->globalref->name)); } int istype = f && jl_is_type(f); if (!istype) { - if (kind == PARTITION_KIND_IMPLICIT) { - check_safe_newbinding(m, var); - return b; + if (jl_bkind_is_some_implicit(kind)) { + check_safe_newbinding(b->globalref->mod, b->globalref->name); + return NULL; } else if (kind != PARTITION_KIND_IMPORTED) { // TODO: we might want to require explicitly importing types to add constructors // or we might want to drop this error entirely - jl_module_t *from = jl_binding_dbgmodule(b, m, var); + jl_module_t *from = jl_binding_dbgmodule(b); + assert(from); // Can only be NULL if implicit, which we excluded above jl_errorf("invalid method definition in %s: function %s.%s must be explicitly imported to be extended", - jl_module_debug_name(m), jl_module_debug_name(from), jl_symbol_name(var)); + jl_module_debug_name(b->globalref->mod), jl_module_debug_name(from), jl_symbol_name(b->globalref->name)); } } else if (kind != PARTITION_KIND_IMPORTED) { - int should_error = strcmp(jl_symbol_name(var), "=>") == 0; - jl_module_t *from = jl_binding_dbgmodule(b, m, var); + int should_error = strcmp(jl_symbol_name(b->globalref->name), "=>") == 0; + jl_module_t *from = jl_binding_dbgmodule(b); if (should_error) { jl_errorf("invalid method definition in %s: function %s.%s must be explicitly imported to be extended", - jl_module_debug_name(m), jl_module_debug_name(from), jl_symbol_name(var)); + jl_module_debug_name(b->globalref->mod), from ? jl_module_debug_name(from) : "", jl_symbol_name(b->globalref->name)); } else if (!(jl_atomic_fetch_or_relaxed(&b->flags, BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION) & BINDING_FLAG_DID_PRINT_IMPLICIT_IMPORT_ADMONITION)) { @@ -868,25 +990,14 @@ JL_DLLEXPORT jl_binding_t *jl_get_binding_for_method_def(jl_module_t *m, jl_sym_ " NOTE: Assumed \"%s\" refers to `%s.%s`. This behavior is deprecated and may differ in future versions.`\n" " NOTE: This behavior may have differed in Julia versions prior to 1.12.\n" " Hint: If you intended to create a new generic function of the same name, use `function %s end`.\n" - " Hint: To silence the warning, qualify `%s` as `%s.%s` or explicitly `import %s: %s`\n", - jl_symbol_name(var), jl_module_debug_name(m), - jl_symbol_name(var), jl_module_debug_name(from), jl_symbol_name(var), - jl_symbol_name(var), jl_symbol_name(var), jl_module_debug_name(from), jl_symbol_name(var), - jl_module_debug_name(from), jl_symbol_name(var)); + " Hint: To silence the warning, qualify `%s` as `%s.%s` in the method signature or explicitly `import %s: %s`.\n", + jl_symbol_name(b->globalref->name), jl_module_debug_name(b->globalref->mod), + jl_symbol_name(b->globalref->name), jl_module_debug_name(from), jl_symbol_name(b->globalref->name), + jl_symbol_name(b->globalref->name), jl_symbol_name(b->globalref->name), jl_module_debug_name(from), jl_symbol_name(b->globalref->name), + jl_module_debug_name(from), jl_symbol_name(b->globalref->name)); } } - return ownerb; -} - -// for error message printing: look up the module that exported a binding to m as var -// this might not be the same as the owner of the binding, since the binding itself may itself have been imported from elsewhere -static jl_module_t *jl_binding_dbgmodule(jl_binding_t *b, jl_module_t *m, jl_sym_t *var) -{ - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (jl_bkind_is_some_import(jl_binding_kind(bpart))) { - return ((jl_binding_t*)bpart->restriction)->globalref->mod; - } - return m; + return f; } static void jl_binding_dep_message(jl_binding_t *b); @@ -995,6 +1106,22 @@ JL_DLLEXPORT void check_safe_import_from(jl_module_t *m) } } +static int eq_bindings(jl_binding_partition_t *owner, jl_binding_t *alias, size_t world) +{ + jl_binding_t *ownerb = NULL; + jl_binding_partition_t *alias_bpart = jl_get_binding_partition(alias, world); + if (owner == alias_bpart) + return 1; + jl_walk_binding_inplace(&ownerb, &owner, world); + jl_walk_binding_inplace(&alias, &alias_bpart, world); + if (jl_bkind_is_some_constant(jl_binding_kind(owner)) && + jl_bkind_is_some_constant(jl_binding_kind(alias_bpart)) && + owner->restriction && + alias_bpart->restriction == owner->restriction) + return 1; + return owner == alias_bpart; +} + // NOTE: we use explici since explicit is a C++ keyword static void module_import_(jl_task_t *ct, jl_module_t *to, jl_module_t *from, jl_sym_t *asname, jl_sym_t *s, int explici) { @@ -1148,7 +1275,7 @@ JL_DLLEXPORT void jl_module_using(jl_module_t *to, jl_module_t *from) if (tobpart) { enum jl_partition_kind kind = jl_binding_kind(tobpart); if (jl_bkind_is_some_implicit(kind)) { - jl_replace_binding_locked(tob, tobpart, NULL, PARTITION_KIND_IMPLICIT_RECOMPUTE, new_world); + jl_replace_binding_locked(tob, tobpart, NULL, PARTITION_FAKE_KIND_IMPLICIT_RECOMPUTE, new_world); } } } @@ -1437,10 +1564,12 @@ JL_DLLEXPORT jl_binding_partition_t *jl_replace_binding_locked2(jl_binding_t *b, jl_binding_partition_t *new_bpart = new_binding_partition(); JL_GC_PUSH1(&new_bpart); new_bpart->min_world = new_world; - if ((kind & PARTITION_MASK_KIND) == PARTITION_KIND_IMPLICIT_RECOMPUTE) { + if ((kind & PARTITION_MASK_KIND) == PARTITION_FAKE_KIND_IMPLICIT_RECOMPUTE) { assert(!restriction_val); - jl_check_new_binding_implicit(new_bpart, b, NULL, new_world); - new_bpart->kind |= kind & PARTITION_MASK_FLAG; + struct implicit_search_resolution resolution = jl_resolve_implicit_import(b, NULL, new_world, 0); + new_bpart->kind = resolution.ultimate_kind | (kind & PARTITION_MASK_FLAG); + new_bpart->restriction = resolution.binding_or_const; + assert(resolution.min_world <= new_world && resolution.max_world == ~(size_t)0); if (new_bpart->kind == old_bpart->kind && new_bpart->restriction == old_bpart->restriction) { JL_GC_POP(); return old_bpart; @@ -1561,14 +1690,9 @@ static int should_depwarn(jl_binding_t *b, uint8_t flag) // (`using` or `import`). The logic here is that the thing that needs to be adjusted // is not the use itself, but rather the `using` or `import` (which already prints // an appropriate warning). - for (;;) { - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (bpart->kind & PARTITION_FLAG_DEPRECATED) - return 1; - if ((bpart->kind & PARTITION_MASK_KIND) != PARTITION_KIND_IMPLICIT) - break; - b = (jl_binding_t*)bpart->restriction; - } + jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); + if (bpart->kind & flag) + return 1; return 0; } @@ -1592,18 +1716,6 @@ void jl_binding_deprecation_warning(jl_binding_t *b) jl_printf(JL_STDERR, "WARNING: "); jl_printf(JL_STDERR, "Use of "); - jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - int first = 0; - while (!(bpart->kind & PARTITION_FLAG_DEPWARN)) { - if (first) { - jl_printf(JL_STDERR, "binding implicitly imported via "); - first = 0; - } - jl_printf(JL_STDERR, "%s.%s -> ", jl_symbol_name(b->globalref->mod->name), jl_symbol_name(b->globalref->name)); - assert(jl_binding_kind(bpart) == PARTITION_KIND_IMPLICIT); - b = (jl_binding_t*)bpart->restriction; - bpart = jl_get_binding_partition(b, jl_current_task->world_age); - } jl_printf(JL_STDERR, "%s.%s is deprecated", jl_symbol_name(b->globalref->mod->name), jl_symbol_name(b->globalref->name)); jl_binding_dep_message(b); @@ -1807,7 +1919,7 @@ JL_DLLEXPORT void jl_clear_implicit_imports(jl_module_t *m) if ((void*)b == jl_nothing) break; jl_binding_partition_t *bpart = jl_get_binding_partition(b, jl_current_task->world_age); - if (jl_binding_kind(bpart) == PARTITION_KIND_IMPLICIT) { + if (jl_bkind_is_some_implicit(jl_binding_kind(bpart))) { jl_atomic_store_relaxed(&b->partitions, NULL); } } diff --git a/src/staticdata.c b/src/staticdata.c index 4671b5c1fecf6..62f3feeaa2159 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3561,12 +3561,14 @@ static int jl_validate_binding_partition(jl_binding_t *b, jl_binding_partition_t size_t raw_kind = bpart->kind; enum jl_partition_kind kind = (enum jl_partition_kind)(raw_kind & PARTITION_MASK_KIND); if (!unchanged_implicit && jl_bkind_is_some_implicit(kind)) { - jl_check_new_binding_implicit(bpart, b, NULL, jl_atomic_load_relaxed(&jl_world_counter)); + // TODO: Should we actually update this in place or delete it from the partitions list + // and allocate a fresh bpart? + jl_update_loaded_bpart(b, bpart); bpart->kind |= (raw_kind & PARTITION_MASK_FLAG); if (bpart->min_world > jl_require_world) goto invalidated; } - if (!jl_bkind_is_some_import(kind)) + if (!jl_bkind_is_some_explicit_import(kind) && kind != PARTITION_KIND_IMPLICIT_GLOBAL) return 1; jl_binding_t *imported_binding = (jl_binding_t*)bpart->restriction; if (no_replacement) diff --git a/src/toplevel.c b/src/toplevel.c index 35ec0684ab7a7..73b65ecf2e3fb 100644 --- a/src/toplevel.c +++ b/src/toplevel.c @@ -311,7 +311,7 @@ void jl_declare_global(jl_module_t *m, jl_value_t *arg, jl_value_t *set_type, in bpart = jl_get_binding_partition(b, new_world); enum jl_partition_kind kind = jl_binding_kind(bpart); if (kind != PARTITION_KIND_GLOBAL) { - if (jl_bkind_is_some_guard(kind) || kind == PARTITION_KIND_DECLARED || kind == PARTITION_KIND_IMPLICIT) { + if (jl_bkind_is_some_implicit(kind) || kind == PARTITION_KIND_DECLARED) { if (kind == new_kind) { if (!set_type) goto done; @@ -659,9 +659,9 @@ static void import_module(jl_task_t *ct, jl_module_t *JL_NONNULL m, jl_module_t jl_binding_t *b = jl_get_module_binding(m, name, 1); jl_binding_partition_t *bpart = jl_get_binding_partition(b, ct->world_age); enum jl_partition_kind kind = jl_binding_kind(bpart); - if (kind != PARTITION_KIND_GUARD && kind != PARTITION_KIND_FAILED && kind != PARTITION_KIND_DECLARED && kind != PARTITION_KIND_IMPLICIT) { + if (!jl_bkind_is_some_implicit(kind) && kind != PARTITION_KIND_DECLARED) { // Unlike regular constant declaration, we allow this as long as we eventually end up at a constant. - jl_walk_binding_inplace(&b, &bpart, ct->world_age); + jl_walk_binding_inplace(&b, &bpart, ct->world_age); if (jl_binding_kind(bpart) == PARTITION_KIND_CONST || jl_binding_kind(bpart) == PARTITION_KIND_BACKDATED_CONST || jl_binding_kind(bpart) == PARTITION_KIND_CONST_IMPORT) { // Already declared (e.g. on another thread) or imported. if (bpart->restriction == (jl_value_t*)import) diff --git a/test/core.jl b/test/core.jl index 9a90e97ccf950..97601e7c0c21a 100644 --- a/test/core.jl +++ b/test/core.jl @@ -6120,7 +6120,6 @@ module GlobalDef18933 global sincos nothing end - @test which(@__MODULE__, :sincos) === Base.Math @test @isdefined sincos @test sincos === Base.sincos end @@ -8463,3 +8462,54 @@ module GlobalAssign57446 (@__MODULE__).theglobal = 1 @test theglobal == 1 end + +# issue #57638 - circular imports +module M57638 +module I + using ..M57638 +end +using .I +end +convert(Core.Binding, GlobalRef(M57638.I, :Base)) +@test M57638.Base === Base + +module M57638_2 +module I + using ..M57638_2 + export Base +end +using .I +export Base +end +@test M57638_2.Base === Base + +module M57638_3 + module M2 + using ..M57638_3 + module M3 + const x = 1 + export x + end + using .M3 + export x + end + using .M2 + export x +end +@test M57638_3.x === 1 + +module GlobalBindingMulti + module M + export S + module C + export S + struct A end + S = A() # making S const makes the error go away + end + using .C + end + + using .M + using .M.C +end +@test GlobalBindingMulti.S === GlobalBindingMulti.M.C.S diff --git a/test/reflection.jl b/test/reflection.jl index 3752bd9c56c88..345e219b41a3e 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -1295,3 +1295,24 @@ end @test Base.infer_return_type(code_lowered, (Any,Any)) == Vector{Core.CodeInfo} @test methods(Union{}) == Any[m.method for m in Base._methods_by_ftype(Tuple{Core.TypeofBottom, Vararg}, 1, Base.get_world_counter())] # issue #55187 + +# which should not look through const bindings, even if they have the same value +# as a previous implicit import +module SinConst +const sin = Base.sin +end + +@test which(SinConst, :sin) === SinConst + +# `which` should error if there is not a unique binding that a constant was imported from +module X1ConstConflict +const xconstconflict = 1 +export xconstconflict +end +module X2ConstConflict +const xconstconflict = 1 +export xconstconflict +end +using .X1ConstConflict, .X2ConstConflict + +@test_throws ErrorException which(@__MODULE__, :xconstconflict)