merge queue: embarking master (beba231), #11450 and #11386 together

Cabal-QuickCheck/src/Test/QuickCheck/Instances/Cabal.hs

@@ -389,6 +389,7 @@ instance Arbitrary a => Arbitrary (InstallDirs a) where
         <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary --  8
         <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary -- 12
         <*> arbitrary <*> arbitrary <*> arbitrary <*> arbitrary -- 16
+        <*> arbitrary                                        -- 17
 
 instance Arbitrary PathTemplate where
     arbitrary = toPathTemplate <$> arbitraryShortToken

Cabal-syntax/src/Distribution/Types/InstalledPackageInfo.hs

@@ -73,6 +73,7 @@ data InstalledPackageInfo = InstalledPackageInfo
   , libraryDirs :: [FilePath]
   , libraryDirsStatic :: [FilePath]
   , libraryDynDirs :: [FilePath]
+  , libraryBytecodeDirs :: [FilePath]
   -- ^ overrides 'libraryDirs'
   , dataDir :: FilePath
   , hsLibraries :: [String]
@@ -160,6 +161,7 @@ emptyInstalledPackageInfo =
     , hsLibraries = []
     , extraLibraries = []
     , extraLibrariesStatic = []
+    , libraryBytecodeDirs = []
     , extraGHCiLibraries = []
     , includeDirs = []
     , includes = []

Cabal-syntax/src/Distribution/Types/InstalledPackageInfo/FieldGrammar.hs

@@ -108,6 +108,7 @@ ipiFieldGrammar =
     <@> monoidalFieldAla "library-dirs" (alaList' FSep FilePathNT) L.libraryDirs
     <@> monoidalFieldAla "library-dirs-static" (alaList' FSep FilePathNT) L.libraryDirsStatic
     <@> monoidalFieldAla "dynamic-library-dirs" (alaList' FSep FilePathNT) L.libraryDynDirs
+    <@> monoidalFieldAla "bytecode-library-dirs" (alaList' FSep FilePathNT) L.libraryBytecodeDirs
     <@> optionalFieldDefAla "data-dir" FilePathNT L.dataDir ""
     <@> monoidalFieldAla "hs-libraries" (alaList' FSep Token) L.hsLibraries
     <@> monoidalFieldAla "extra-libraries" (alaList' FSep Token) L.extraLibraries

Cabal-syntax/src/Distribution/Types/InstalledPackageInfo/Lens.hs

@@ -123,6 +123,10 @@ libraryDynDirs :: Lens' InstalledPackageInfo [FilePath]
 libraryDynDirs f s = fmap (\x -> s{T.libraryDynDirs = x}) (f (T.libraryDynDirs s))
 {-# INLINE libraryDynDirs #-}
 
+libraryBytecodeDirs :: Lens' InstalledPackageInfo [FilePath]
+libraryBytecodeDirs f s = fmap (\x -> s{T.libraryBytecodeDirs = x}) (f (T.libraryBytecodeDirs s))
+{-# INLINE libraryBytecodeDirs #-}
+
 dataDir :: Lens' InstalledPackageInfo FilePath
 dataDir f s = fmap (\x -> s{T.dataDir = x}) (f (T.dataDir s))
 {-# INLINE dataDir #-}

Cabal-tests/tests/ParserTests/ipi/Includes2.expr

@@ -54,6 +54,7 @@ InstalledPackageInfo {
   libraryDynDirs =
   [
     "/home/travis/build/haskell/cabal/cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.dist/work/./dist/build/x86_64-linux/ghc-8.2.2/Includes2-0.1.0.0/l/mylib/Includes2-0.1.0.0-inplace-mylib+3gY9SyjX86dBypHcOaev1n/build/Includes2-0.1.0.0-inplace-mylib+3gY9SyjX86dBypHcOaev1n"],
+  libraryBytecodeDirs = [],
   dataDir =
   "/home/travis/build/haskell/cabal/cabal-testsuite/PackageTests/Backpack/Includes2",
   hsLibraries = [

Cabal-tests/tests/ParserTests/ipi/internal-preprocessor-test.expr

@@ -47,6 +47,7 @@ InstalledPackageInfo {
     "/home/ogre/Documents/other-haskell/cabal/cabal-testsuite/PackageTests/CustomPreProcess/setup.dist/work/dist/build"],
   libraryDirsStatic = [],
   libraryDynDirs = [],
+  libraryBytecodeDirs = [],
   dataDir =
   "/home/ogre/Documents/other-haskell/cabal/cabal-testsuite/PackageTests/CustomPreProcess",
   hsLibraries = [

Cabal-tests/tests/ParserTests/ipi/issue-2276-ghc-9885.expr

@@ -153,6 +153,7 @@ InstalledPackageInfo {
   libraryDirsStatic = [],
   libraryDynDirs = [
     "/opt/ghc/8.2.2/lib/ghc-8.2.2/transformers-0.5.2.0"],
+  libraryBytecodeDirs = [],
   dataDir =
   "/opt/ghc/8.2.2/share/x86_64-linux-ghc-8.2.2/transformers-0.5.2.0",
   hsLibraries = [

Cabal-tests/tests/ParserTests/ipi/transformers.expr

@@ -153,6 +153,7 @@ InstalledPackageInfo {
   libraryDirsStatic = [],
   libraryDynDirs = [
     "/opt/ghc/8.2.2/lib/ghc-8.2.2/transformers-0.5.2.0"],
+  libraryBytecodeDirs = [],
   dataDir =
   "/opt/ghc/8.2.2/share/x86_64-linux-ghc-8.2.2/transformers-0.5.2.0",
   hsLibraries = [

Cabal-tests/tests/UnitTests/Distribution/Utils/Structured.hs

@@ -33,5 +33,4 @@ md5CheckGenericPackageDescription proxy = md5Check proxy
 
 md5CheckLocalBuildInfo :: Proxy LocalBuildInfo -> Assertion
 md5CheckLocalBuildInfo proxy = md5Check proxy
-    0xe9c033a98273061c4a4cc8f9653193a2
-
+    0xe38f63a643a5782e0ee7d16453796142

Cabal/src/Distribution/Simple/BuildPaths.hs

@@ -42,6 +42,7 @@ module Distribution.Simple.BuildPaths
   , mkSharedLibName
   , mkProfSharedLibName
   , mkStaticLibName
+  , mkBytecodeLibName
   , mkGenericSharedBundledLibName
   , exeExtension
   , objExtension
@@ -409,6 +410,9 @@ mkStaticLibName platform (CompilerId compilerFlavor compilerVersion) lib =
   where
     comp = prettyShow compilerFlavor ++ prettyShow compilerVersion
 
+mkBytecodeLibName :: CompilerId -> UnitId -> String
+mkBytecodeLibName _comp lib = getHSLibraryName lib <.> ".bytecodelib"
+
 -- | Create a library name for a bundled shared library from a given name.
 -- This matches the naming convention for shared libraries as implemented in
 -- GHC's packageHsLibs function in the Packages module.

Cabal/src/Distribution/Simple/BuildWay.hs

@@ -1,14 +1,21 @@
 {-# LANGUAGE LambdaCase #-}
 
-module Distribution.Simple.BuildWay where
+module Distribution.Simple.BuildWay
+  ( BuildWay (..)
+  , buildWayObjectExtension
+  , buildWayInterfaceExtension
+  ) where
 
 data BuildWay = StaticWay | DynWay | ProfWay | ProfDynWay
   deriving (Eq, Ord, Show, Read, Enum)
 
--- | Returns the object/interface extension prefix for the given build way (e.g. "dyn_" for 'DynWay')
-buildWayPrefix :: BuildWay -> String
-buildWayPrefix = \case
-  StaticWay -> ""
-  ProfWay -> "p_"
-  DynWay -> "dyn_"
-  ProfDynWay -> "p_dyn_"
+-- | Returns the object extension for the given build way (e.g. "dyn_o" for 'DynWay' on ELF)
+buildWayObjectExtension :: String -> BuildWay -> String
+buildWayObjectExtension ext = \case
+  StaticWay -> ext
+  ProfWay -> "p_" ++ ext
+  DynWay -> "dyn_" ++ ext
+  ProfDynWay -> "p_dyn_" ++ ext
+
+buildWayInterfaceExtension :: BuildWay -> String
+buildWayInterfaceExtension = buildWayObjectExtension "hi"

Cabal/src/Distribution/Simple/Compiler.hs

@@ -79,6 +79,7 @@ module Distribution.Simple.Compiler
   , profilingVanillaSupported
   , profilingVanillaSupportedOrUnknown
   , dynamicSupported
+  , bytecodeArtifactsSupported
   , backpackSupported
   , arResponseFilesSupported
   , arDashLSupported
@@ -573,6 +574,14 @@ profilingDynamicSupportedOrUnknown comp =
 dynamicSupported :: Compiler -> Maybe Bool
 dynamicSupported comp = waySupported "dyn" comp
 
+-- | Does the compiler support producing bytecode objects and bytecode libraries?
+bytecodeArtifactsSupported :: Compiler -> Bool
+bytecodeArtifactsSupported comp
+  | GHC <- compilerFlavor comp
+  , compilerVersion comp >= mkVersion [9, 15, 0] =
+      True
+  | otherwise = False
+
 -- | Does this compiler support a package database entry with:
 -- "visibility"?
 libraryVisibilitySupported :: Compiler -> Bool

Cabal/src/Distribution/Simple/Configure.hs

@@ -762,6 +762,10 @@ computeLocalBuildConfig verbHandles cfg comp programDb = do
         -- into a huge .a archive) via GHCs -staticlib flag.
         fromFlagOrDefault False $ configStaticLib cfg
 
+      -- See doc/internal/bytecode-libraries.md for the end-to-end story.
+      withBytecodeLib_ =
+        fromFlagOrDefault False $ configBytecodeLib cfg
+
       withDynExe_ = fromFlag $ configDynExe cfg
 
       withFullyStaticExe_ = fromFlag $ configFullyStaticExe cfg
@@ -789,12 +793,21 @@ computeLocalBuildConfig verbHandles cfg comp programDb = do
   strip_lib <- strip_libexe "library" configStripLibs
   strip_exe <- strip_libexe "executable" configStripExes
 
+  checkedWithBytecodeLib <-
+    if bytecodeArtifactsSupported comp
+      then return withBytecodeLib_
+      else do
+        when withBytecodeLib_ $
+          warn verbosity "This compiler does not support bytecode libraries; ignoring --enable-library-bytecode"
+        return False
+
   let buildOptions =
         setCoverage . setProfiling $
           LBC.BuildOptions
             { withVanillaLib = fromFlag $ configVanillaLib cfg
             , withSharedLib = withSharedLib_
             , withStaticLib = withStaticLib_
+            , withBytecodeLib = checkedWithBytecodeLib
             , withDynExe = withDynExe_
             , withFullyStaticExe = withFullyStaticExe_
             , withProfLib = False
@@ -1376,6 +1389,7 @@ configureComponents
       dirinfo "Executables" (bindir dirs) (bindir relative)
       dirinfo "Libraries" (libdir dirs) (libdir relative)
       dirinfo "Dynamic Libraries" (dynlibdir dirs) (dynlibdir relative)
+      dirinfo "Bytecode Libraries" (bytecodelibdir dirs) (bytecodelibdir relative)
       dirinfo "Private executables" (libexecdir dirs) (libexecdir relative)
       dirinfo "Data files" (datadir dirs) (datadir relative)
       dirinfo "Documentation" (docdir dirs) (docdir relative)
@@ -2856,6 +2870,7 @@ checkRelocatable verbosity pkg lbi =
             [ bindir
             , libdir
             , dynlibdir
+            , bytecodelibdir
             , libexecdir
             , includedir
             , datadir

Cabal/src/Distribution/Simple/Errors.hs

@@ -170,6 +170,7 @@ data CabalException
   | MissingCoveredInstalledLibrary UnitId
   | SetupHooksException SetupHooksException
   | MultiReplDoesNotSupportComplexReexportedModules PackageName ComponentName
+  | StandaloneBytecodeNotSupportedYet
   deriving (Show)
 
 exceptionCode :: CabalException -> Int
@@ -304,6 +305,7 @@ exceptionCode e = case e of
   SetupHooksException err ->
     setupHooksExceptionCode err
   MultiReplDoesNotSupportComplexReexportedModules{} -> 9355
+  StandaloneBytecodeNotSupportedYet -> 9356
 
 versionRequirement :: VersionRange -> String
 versionRequirement range
@@ -804,3 +806,8 @@ exceptionMessage e = case e of
       ++ prettyShow pname
       ++ " a module renaming was found.\n"
       ++ "Multi-repl does not work with complicated reexported-modules until GHC-9.12."
+  StandaloneBytecodeNotSupportedYet ->
+    unlines
+      [ "The ecosystem doesn't support building packages with just bytecode libraries yet."
+      , "If you have run into this error, please comment on issue #<TODO>"
+      ]

Cabal/src/Distribution/Simple/GHC.hs

@@ -280,25 +280,6 @@ compilerProgramDb
   -- ^ user-specified @ghc-pkg@ path (optional)
   -> IO ProgramDb
 compilerProgramDb verbosity comp progdb1 hcPkgPath = do
-  let
-    ghcProg = fromJust $ lookupProgram ghcProgram progdb1
-    ghcVersion = compilerVersion comp
-
-  -- This is slightly tricky, we have to configure ghc first, then we use the
-  -- location of ghc to help find ghc-pkg in the case that the user did not
-  -- specify the location of ghc-pkg directly:
-  (ghcPkgProg, ghcPkgVersion, progdb2) <-
-    requireProgramVersion
-      verbosity
-      ghcPkgProgram
-        { programFindLocation = guessGhcPkgFromGhcPath ghcProg
-        }
-      anyVersion
-      (userMaybeSpecifyPath "ghc-pkg" hcPkgPath progdb1)
-
-  when (ghcVersion /= ghcPkgVersion) $
-    dieWithException verbosity $
-      VersionMismatchGHC (programPath ghcProg) ghcVersion (programPath ghcPkgProg) ghcPkgVersion
   -- Likewise we try to find the matching hsc2hs and haddock programs.
   let hsc2hsProgram' =
         hsc2hsProgram
@@ -316,20 +297,49 @@ compilerProgramDb verbosity comp progdb1 hcPkgPath = do
         runghcProgram
           { programFindLocation = guessRunghcFromGhcPath ghcProg
           }
-      progdb3 =
-        addKnownProgram haddockProgram' $
-          addKnownProgram hsc2hsProgram' $
-            addKnownProgram hpcProgram' $
-              addKnownProgram runghcProgram' progdb2
+      ghcPkgProgram' =
+        ghcPkgProgram
+          { programFindLocation = guessGhcPkgFromGhcPath ghcProg
+          }
+      progdb2 =
+        -- The knownPrograms are populated before userMaybeSpecifyPath
+        -- in the case that ProgramDb has been restored from a cache and is empty
+        -- See #11373 for where this went wrong before
+        userMaybeSpecifyPath "ghc-pkg" hcPkgPath $
+          addKnownProgram haddockProgram' $
+            addKnownProgram hsc2hsProgram' $
+              addKnownProgram hpcProgram' $
+                addKnownProgram runghcProgram' $
+                  addKnownProgram
+                    ghcPkgProgram'
+                    progdb1
+
+      ghcProg = fromJust $ lookupProgram ghcProgram progdb1
+      ghcVersion = compilerVersion comp
 
       -- configure gcc, ld, ar etc... based on the paths stored
       -- in the GHC settings file
-      progdb4 =
+      progdb3 =
         Internal.configureToolchain
           (ghcVersionImplInfo ghcVersion)
           ghcProg
           (compilerProperties comp)
-          progdb3
+          progdb2
+
+  -- This is slightly tricky, we have to configure ghc first, then we use the
+  -- location of ghc to help find ghc-pkg in the case that the user did not
+  -- specify the location of ghc-pkg directly:
+  (ghcPkgProg, ghcPkgVersion, progdb4) <-
+    requireProgramVersion
+      verbosity
+      ghcPkgProgram'
+      anyVersion
+      progdb3
+
+  when (ghcVersion /= ghcPkgVersion) $
+    dieWithException verbosity $
+      VersionMismatchGHC (programPath ghcProg) ghcVersion (programPath ghcPkgProg) ghcPkgVersion
+
   return progdb4
 
 -- | Given something like /usr/local/bin/ghc-6.6.1(.exe) we try and find
@@ -952,7 +962,7 @@ installFLib verbosity lbi targetDir builtDir _pkg flib =
         nm = unUnqualComponentName $ foreignLibName flib
 #endif /* mingw32_HOST_OS */
 
--- | Install for ghc, .hi, .a and, if --with-ghci given, .o
+-- | Install for ghc, .hi, .a, .so, .bytecodelib and, if --with-ghci given, .o
 installLib
   :: Verbosity
   -> LocalBuildInfo
@@ -961,12 +971,14 @@ installLib
   -> FilePath
   -- ^ install location for dynamic libraries
   -> FilePath
+  -- ^ install location for bytecode libraries
+  -> FilePath
   -- ^ Build location
   -> PackageDescription
   -> Library
   -> ComponentLocalBuildInfo
   -> IO ()
-installLib verbosity lbi targetDir dynlibTargetDir _builtDir pkg lib clbi = do
+installLib verbosity lbi targetDir dynlibTargetDir bytecodeTargetDir _builtDir pkg lib clbi = do
   let
     (wantedLibWays, _, _) = buildWays lbi
     isIndef = componentIsIndefinite clbi
@@ -986,6 +998,10 @@ installLib verbosity lbi targetDir dynlibTargetDir _builtDir pkg lib clbi = do
 
   -- copy the built library files over:
   when (has_code && hasLib) $ do
+    -- Bytecode libraries are installed in bytecodelibdir and are copied
+    -- without stripping; see doc/internal/bytecode-libraries.md.
+    whenBytecodeLib $ installOrdinaryNoStrip builtDir bytecodeTargetDir bytecodeLibName
+
     forM_ libWays $ \w -> case w of
       StaticWay -> do
         sequence_
@@ -1059,7 +1075,7 @@ installLib verbosity lbi targetDir dynlibTargetDir _builtDir pkg lib clbi = do
     builtDir = componentBuildDir lbi clbi
     mbWorkDir = mbWorkDirLBI lbi
 
-    install isShared srcDir dstDir name = do
+    install isShared shouldStrip srcDir dstDir name = do
       let src = i $ srcDir </> makeRelativePathEx name
           dst = dstDir </> name
 
@@ -1069,15 +1085,16 @@ installLib verbosity lbi targetDir dynlibTargetDir _builtDir pkg lib clbi = do
         then installExecutableFile verbosity src dst
         else installOrdinaryFile verbosity src dst
 
-      when (stripLibs lbi) $
+      when (shouldStrip && stripLibs lbi) $
         Strip.stripLib
           verbosity
           platform
           (withPrograms lbi)
           dst
 
-    installOrdinary = install False
-    installShared = install True
+    installOrdinary = install False True
+    installOrdinaryNoStrip = install False False
+    installShared = install True True
 
     copyModuleFiles ext = do
       files <- findModuleFilesCwd verbosity mbWorkDir [builtDir] [ext] (allLibModules lib clbi)
@@ -1095,6 +1112,7 @@ installLib verbosity lbi targetDir dynlibTargetDir _builtDir pkg lib clbi = do
     platform = hostPlatform lbi
     uid = componentUnitId clbi
     profileLibName = mkProfLibName uid
+    bytecodeLibName = mkBytecodeLibName compiler_id uid
     ghciLibName = Internal.mkGHCiLibName uid
     ghciProfLibName = Internal.mkGHCiProfLibName uid
 
@@ -1111,6 +1129,7 @@ installLib verbosity lbi targetDir dynlibTargetDir _builtDir pkg lib clbi = do
       _ -> False
     has_code = not (componentIsIndefinite clbi)
     whenGHCi = when (hasLib && withGHCiLib lbi && has_code)
+    whenBytecodeLib = when (hasLib && withBytecodeLib lbi && has_code)
 
 -- -----------------------------------------------------------------------------
 -- Registering