Introduce Best Effort compilation options

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -21,6 +21,8 @@ class GenBCode extends Phase { self =>
 
   override def description: String = GenBCode.description
 
+  override def isRunnable(using Context) = super.isRunnable && !ctx.usedBestEffortTasty
+
   private val superCallsMap = new MutableSymbolMap[Set[ClassSymbol]]
   def registerSuperCall(sym: Symbol, calls: ClassSymbol): Unit = {
     val old = superCallsMap.getOrElse(sym, Set.empty)

compiler/src/dotty/tools/backend/sjs/GenSJSIR.scala

@@ -12,7 +12,7 @@ class GenSJSIR extends Phase {
   override def description: String = GenSJSIR.description
 
   override def isRunnable(using Context): Boolean =
-    super.isRunnable && ctx.settings.scalajs.value
+    super.isRunnable && ctx.settings.scalajs.value && !ctx.usedBestEffortTasty
 
   def run(using Context): Unit =
     new JSCodeGen().run()

compiler/src/dotty/tools/dotc/Driver.scala

@@ -39,6 +39,9 @@ class Driver {
       catch
         case ex: FatalError =>
           report.error(ex.getMessage.nn) // signals that we should fail compilation.
+        case ex: Throwable if ctx.usedBestEffortTasty =>
+          report.bestEffortError(ex, "Some best-effort tasty files were not able to be read.")
+          throw ex
         case ex: TypeError if !runOrNull.enrichedErrorMessage =>
           println(runOrNull.enrichErrorMessage(s"${ex.toMessage} while compiling ${files.map(_.path).mkString(", ")}"))
           throw ex
@@ -102,8 +105,8 @@ class Driver {
             None
           else file.ext match
             case FileExtension.Jar => Some(file.path)
-            case FileExtension.Tasty =>
-              TastyFileUtil.getClassPath(file) match
+            case FileExtension.Tasty | FileExtension.Betasty =>
+              TastyFileUtil.getClassPath(file, ctx.withBestEffortTasty) match
                 case Some(classpath) => Some(classpath)
                 case _ =>
                   report.error(em"Could not load classname from: ${file.path}")

compiler/src/dotty/tools/dotc/Run.scala

@@ -329,9 +329,13 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
       val profiler = ctx.profiler
       var phasesWereAdjusted = false
 
+      var forceReachPhaseMaybe =
+        if (ctx.isBestEffort && phases.exists(_.phaseName == "typer")) Some("typer")
+        else None
+
       for phase <- allPhases do
         doEnterPhase(phase)
-        val phaseWillRun = phase.isRunnable
+        val phaseWillRun = phase.isRunnable || forceReachPhaseMaybe.nonEmpty
         if phaseWillRun then
           Stats.trackTime(s"phase time ms/$phase") {
             val start = System.currentTimeMillis
@@ -344,6 +348,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
                 def printCtx(unit: CompilationUnit) = phase.printingContext(
                   ctx.fresh.setPhase(phase.next).setCompilationUnit(unit))
                 lastPrintedTree = printTree(lastPrintedTree)(using printCtx(unit))
+
+            if forceReachPhaseMaybe.contains(phase.phaseName) then
+              forceReachPhaseMaybe = None
+
             report.informTime(s"$phase ", start)
             Stats.record(s"total trees at end of $phase", ast.Trees.ntrees)
             for (unit <- units)

compiler/src/dotty/tools/dotc/ast/TreeInfo.scala

@@ -919,12 +919,12 @@ trait TypedTreeInfo extends TreeInfo[Type] { self: Trees.Instance[Type] =>
       else cpy.PackageDef(tree)(pid, slicedStats) :: Nil
     case tdef: TypeDef =>
       val sym = tdef.symbol
-      assert(sym.isClass)
+      assert(sym.isClass || ctx.tolerateErrorsForBestEffort)
       if (cls == sym || cls == sym.linkedClass) tdef :: Nil
       else Nil
     case vdef: ValDef =>
       val sym = vdef.symbol
-      assert(sym.is(Module))
+      assert(sym.is(Module) || ctx.tolerateErrorsForBestEffort)
       if (cls == sym.companionClass || cls == sym.moduleClass) vdef :: Nil
       else Nil
     case tree =>

compiler/src/dotty/tools/dotc/ast/tpd.scala

@@ -47,7 +47,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply | _: Inlined | _: Hole =>
       ta.assignType(untpd.Apply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported)
+      assert(ctx.reporter.errorsReported || ctx.tolerateErrorsForBestEffort)
       ta.assignType(untpd.Apply(fn, args), fn, args)
 
   def TypeApply(fn: Tree, args: List[Tree])(using Context): TypeApply = fn match
@@ -56,7 +56,7 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
     case _: RefTree | _: GenericApply =>
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
     case _ =>
-      assert(ctx.reporter.errorsReported, s"unexpected tree for type application: $fn")
+      assert(ctx.reporter.errorsReported || ctx.tolerateErrorsForBestEffort, s"unexpected tree for type application: $fn")
       ta.assignType(untpd.TypeApply(fn, args), fn, args)
 
   def Literal(const: Constant)(using Context): Literal =

compiler/src/dotty/tools/dotc/classpath/DirectoryClassPath.scala

@@ -285,7 +285,7 @@ case class DirectoryClassPath(dir: JFile) extends JFileDirectoryLookup[BinaryFil
   protected def createFileEntry(file: AbstractFile): BinaryFileEntry = BinaryFileEntry(file)
 
   protected def isMatchingFile(f: JFile): Boolean =
-    f.isTasty || (f.isClass && !f.hasSiblingTasty)
+    f.isTasty || f.isBestEffortTasty || (f.isClass && !f.hasSiblingTasty)
 
   private[dotty] def classes(inPackage: PackageName): Seq[BinaryFileEntry] = files(inPackage)
 }

compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

@@ -23,8 +23,12 @@ object FileUtils {
 
     def hasTastyExtension: Boolean = file.ext.isTasty
 
+    def hasBetastyExtension: Boolean = file.ext.isBetasty
+
     def isTasty: Boolean = !file.isDirectory && hasTastyExtension
 
+    def isBestEffortTasty: Boolean = !file.isDirectory && hasBetastyExtension
+
     def isScalaBinary: Boolean = file.isClass || file.isTasty
 
     def isScalaOrJavaSource: Boolean = !file.isDirectory && file.ext.isScalaOrJava
@@ -55,6 +59,9 @@ object FileUtils {
 
     def isTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_TASTY)
 
+    def isBestEffortTasty: Boolean = file.isFile && file.getName.endsWith(SUFFIX_BETASTY)
+
+
     /**
      * Returns if there is an existing sibling `.tasty` file.
      */
@@ -69,6 +76,7 @@ object FileUtils {
   private val SUFFIX_CLASS = ".class"
   private val SUFFIX_SCALA = ".scala"
   private val SUFFIX_TASTY = ".tasty"
+  private val SUFFIX_BETASTY = ".betasty"
   private val SUFFIX_JAVA = ".java"
   private val SUFFIX_SIG = ".sig"
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -414,6 +414,9 @@ private sealed trait YSettings:
   val YprofileRunGcBetweenPhases: Setting[List[String]] = PhasesSetting(ForkSetting, "Yprofile-run-gc", "Run a GC between phases - this allows heap size to be accurate at the expense of more time. Specify a list of phases, or *", "_")
       //.withPostSetHook( _ => YprofileEnabled.value = true )
 
+  val YbestEffort: Setting[Boolean] = BooleanSetting(ForkSetting, "Ybest-effort", "Enable best-effort compilation attempting to produce betasty to the META-INF/best-effort directory, regardless of errors, as part of the pickler phase.")
+  val YwithBestEffortTasty: Setting[Boolean] = BooleanSetting(ForkSetting, "Ywith-best-effort-tasty", "Allow to compile using best-effort tasty files. If such file is used, the compiler will stop after the pickler phase.")
+
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-kind-polymorphism", "Disable kind polymorphism.")
   val YexplicitNulls: Setting[Boolean] = BooleanSetting(ForkSetting, "Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -474,6 +474,21 @@ object Contexts {
 
     /** Is the flexible types option set? */
     def flexibleTypes: Boolean = base.settings.YexplicitNulls.value && !base.settings.YnoFlexibleTypes.value
+
+    /** Is the best-effort option set? */
+    def isBestEffort: Boolean = base.settings.YbestEffort.value
+
+    /** Is the with-best-effort-tasty option set? */
+    def withBestEffortTasty: Boolean = base.settings.YwithBestEffortTasty.value
+
+    /** Were any best effort tasty dependencies used during compilation? */
+    def usedBestEffortTasty: Boolean = base.usedBestEffortTasty
+
+    /** Confirm that a best effort tasty dependency was used during compilation. */
+    def setUsedBestEffortTasty(): Unit = base.usedBestEffortTasty = true
+
+    /** Is either the best-effort option set or .betasty files were used during compilation? */
+    def tolerateErrorsForBestEffort = isBestEffort || usedBestEffortTasty
 
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
@@ -960,6 +975,9 @@ object Contexts {
     val sources: util.HashMap[AbstractFile, SourceFile] = util.HashMap[AbstractFile, SourceFile]()
     val files: util.HashMap[TermName, AbstractFile] = util.HashMap()
 
+    /** Was best effort file used during compilation? */
+    private[core] var usedBestEffortTasty = false
+
     // Types state
     /** A table for hash consing unique types */
     private[core] val uniques: Uniques = Uniques()

compiler/src/dotty/tools/dotc/core/DenotTransformers.scala

@@ -28,6 +28,8 @@ object DenotTransformers {
 
     /** The transformation method */
     def transform(ref: SingleDenotation)(using Context): SingleDenotation
+
+    override def isRunnable(using Context) = super.isRunnable && !ctx.usedBestEffortTasty
   }
 
   /** A transformer that only transforms the info field of denotations */

compiler/src/dotty/tools/dotc/core/Denotations.scala

@@ -719,7 +719,8 @@ object Denotations {
         ctx.runId >= validFor.runId
         || ctx.settings.YtestPickler.value // mixing test pickler with debug printing can travel back in time
         || ctx.mode.is(Mode.Printing)  // no use to be picky when printing error messages
-        || symbol.isOneOf(ValidForeverFlags),
+        || symbol.isOneOf(ValidForeverFlags)
+        || ctx.tolerateErrorsForBestEffort,
         s"denotation $this invalid in run ${ctx.runId}. ValidFor: $validFor")
       var d: SingleDenotation = this
       while ({

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -720,12 +720,16 @@ object SymDenotations {
      *  TODO: Find a more robust way to characterize self symbols, maybe by
      *       spending a Flag on them?
      */
-    final def isSelfSym(using Context): Boolean = owner.infoOrCompleter match {
-      case ClassInfo(_, _, _, _, selfInfo) =>
-        selfInfo == symbol ||
-          selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
-      case _ => false
-    }
+    final def isSelfSym(using Context): Boolean =
+      if !ctx.isBestEffort || exists then
+        owner.infoOrCompleter match {
+          case ClassInfo(_, _, _, _, selfInfo) =>
+            selfInfo == symbol ||
+              selfInfo.isInstanceOf[Type] && name == nme.WILDCARD
+          case _ => false
+        }
+      else false
+
 
     /** Is this definition contained in `boundary`?
      *  Same as `ownersIterator contains boundary` but more efficient.
@@ -2003,7 +2007,7 @@ object SymDenotations {
         case p :: parents1 =>
           p.classSymbol match {
             case pcls: ClassSymbol => builder.addAll(pcls.baseClasses)
-            case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive), s"$this has non-class parent: $p")
+            case _ => assert(isRefinementClass || p.isError || ctx.mode.is(Mode.Interactive) || ctx.tolerateErrorsForBestEffort, s"$this has non-class parent: $p")
           }
           traverse(parents1)
         case nil =>

compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

@@ -7,7 +7,7 @@ import java.nio.channels.ClosedByInterruptException
 
 import scala.util.control.NonFatal
 
-import dotty.tools.dotc.classpath.FileUtils.hasTastyExtension
+import dotty.tools.dotc.classpath.FileUtils.{hasTastyExtension, hasBetastyExtension}
 import dotty.tools.io.{ ClassPath, ClassRepresentation, AbstractFile }
 import dotty.tools.backend.jvm.DottyBackendInterface.symExtensions
 
@@ -26,6 +26,7 @@ import parsing.JavaParsers.OutlineJavaParser
 import parsing.Parsers.OutlineParser
 import dotty.tools.tasty.{TastyHeaderUnpickler, UnpickleException, UnpicklerConfig, TastyVersion}
 import dotty.tools.dotc.core.tasty.TastyUnpickler
+import dotty.tools.tasty.besteffort.BestEffortTastyHeaderUnpickler
 
 object SymbolLoaders {
   import ast.untpd.*
@@ -198,7 +199,7 @@ object SymbolLoaders {
         enterToplevelsFromSource(owner, nameOf(classRep), src)
       case (Some(bin), _) =>
         val completer =
-          if bin.hasTastyExtension then ctx.platform.newTastyLoader(bin)
+          if bin.hasTastyExtension || bin.hasBetastyExtension then ctx.platform.newTastyLoader(bin)
           else ctx.platform.newClassLoader(bin)
         enterClassAndModule(owner, nameOf(classRep), completer)
     }
@@ -261,7 +262,8 @@ object SymbolLoaders {
       (idx + str.TOPLEVEL_SUFFIX.length + 1 != name.length || !name.endsWith(str.TOPLEVEL_SUFFIX))
     }
 
-    def maybeModuleClass(classRep: ClassRepresentation): Boolean = classRep.name.last == '$'
+    def maybeModuleClass(classRep: ClassRepresentation): Boolean =
+      classRep.name.nonEmpty && classRep.name.last == '$'
 
     private def enterClasses(root: SymDenotation, packageName: String, flat: Boolean)(using Context) = {
       def isAbsent(classRep: ClassRepresentation) =
@@ -416,34 +418,45 @@ class ClassfileLoader(val classfile: AbstractFile) extends SymbolLoader {
 }
 
 class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
-
+  val isBestEffortTasty = tastyFile.hasBetastyExtension
   private val unpickler: tasty.DottyUnpickler =
     handleUnpicklingExceptions:
       val tastyBytes = tastyFile.toByteArray
-      new tasty.DottyUnpickler(tastyFile, tastyBytes) // reads header and name table
+      new tasty.DottyUnpickler(tastyFile, tastyBytes, isBestEffortTasty) // reads header and name table
 
   val compilationUnitInfo: CompilationUnitInfo | Null = unpickler.compilationUnitInfo
 
-  def description(using Context): String = "TASTy file " + tastyFile.toString
+  def description(using Context): String =
+    if isBestEffortTasty then "Best Effort TASTy file " + tastyFile.toString
+    else "TASTy file " + tastyFile.toString
 
   override def doComplete(root: SymDenotation)(using Context): Unit =
     handleUnpicklingExceptions:
-      checkTastyUUID()
       val (classRoot, moduleRoot) = rootDenots(root.asClass)
-      unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
-      if mayLoadTreesFromTasty then
-        classRoot.classSymbol.rootTreeOrProvider = unpickler
-        moduleRoot.classSymbol.rootTreeOrProvider = unpickler
+      if (!isBestEffortTasty || ctx.withBestEffortTasty) then
+        val tastyBytes = tastyFile.toByteArray
+        unpickler.enter(roots = Set(classRoot, moduleRoot, moduleRoot.sourceModule))(using ctx.withSource(util.NoSource))
+        if mayLoadTreesFromTasty || isBestEffortTasty then
+          classRoot.classSymbol.rootTreeOrProvider = unpickler
+          moduleRoot.classSymbol.rootTreeOrProvider = unpickler
+        if isBestEffortTasty then
+          checkBeTastyUUID(tastyFile, tastyBytes)
+          ctx.setUsedBestEffortTasty()
+        else
+          checkTastyUUID()
+      else
+        report.error(em"Cannot read Best Effort TASTy $tastyFile without the ${ctx.settings.YwithBestEffortTasty.name} option")
 
   private def handleUnpicklingExceptions[T](thunk: =>T): T =
     try thunk
     catch case e: RuntimeException =>
+      val tastyType = if (isBestEffortTasty) "Best Effort TASTy" else "TASTy"
       val message = e match
         case e: UnpickleException =>
-          s"""TASTy file ${tastyFile.canonicalPath} could not be read, failing with:
+          s"""$tastyType file ${tastyFile.canonicalPath} could not be read, failing with:
             |  ${Option(e.getMessage).getOrElse("")}""".stripMargin
         case _ =>
-          s"""TASTy file ${tastyFile.canonicalPath} is broken, reading aborted with ${e.getClass}
+          s"""$tastyFile file ${tastyFile.canonicalPath} is broken, reading aborted with ${e.getClass}
             |  ${Option(e.getMessage).getOrElse("")}""".stripMargin
       throw IOException(message, e)
 
@@ -460,6 +473,9 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
       // tasty file compiled by `-Yearly-tasty-output-write` comes from an early output jar.
       report.inform(s"No classfiles found for $tastyFile when checking TASTy UUID")
 
+  private def checkBeTastyUUID(tastyFile: AbstractFile, tastyBytes: Array[Byte])(using Context): Unit =
+    new BestEffortTastyHeaderUnpickler(tastyBytes).readHeader()
+
   private def mayLoadTreesFromTasty(using Context): Boolean =
     ctx.settings.YretainTrees.value || ctx.settings.fromTasty.value
 }

compiler/src/dotty/tools/dotc/core/TypeErasure.scala

@@ -747,7 +747,9 @@ class TypeErasure(sourceLanguage: SourceLanguage, semiEraseVCs: Boolean, isConst
     assert(!etp.isInstanceOf[WildcardType] || inSigName, i"Unexpected WildcardType erasure for $tp")
     etp
 
-  /** Like translucentSuperType, but issue a fatal error if it does not exist. */
+  /** Like translucentSuperType, but issue a fatal error if it does not exist.
+   *  If using the best-effort option, the fatal error will not be issued.
+  */
   private def checkedSuperType(tp: TypeProxy)(using Context): Type =
     val tp1 = tp.translucentSuperType
     if !tp1.exists then
@@ -756,7 +758,8 @@ class TypeErasure(sourceLanguage: SourceLanguage, semiEraseVCs: Boolean, isConst
           MissingType(tycon.prefix, tycon.name)
         case _ =>
           TypeError(em"Cannot resolve reference to $tp")
-      throw typeErr
+      if ctx.isBestEffort then report.error(typeErr.toMessage)
+      else throw typeErr
     tp1
 
   /** Widen term ref, skipping any `()` parameter of an eventual getter. Used to erase a TermRef.

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -3149,7 +3149,8 @@ object Types extends TypeUtils {
       if (ctx.erasedTypes) tref
       else cls.info match {
         case cinfo: ClassInfo => cinfo.selfType
-        case _: ErrorType | NoType if ctx.mode.is(Mode.Interactive) => cls.info
+        case _: ErrorType | NoType
+          if ctx.mode.is(Mode.Interactive) || ctx.tolerateErrorsForBestEffort => cls.info
           // can happen in IDE if `cls` is stale
       }
 
@@ -3719,8 +3720,9 @@ object Types extends TypeUtils {
 
     def apply(tp1: Type, tp2: Type, soft: Boolean)(using Context): OrType = {
       def where = i"in union $tp1 | $tp2"
-      expectValueTypeOrWildcard(tp1, where)
-      expectValueTypeOrWildcard(tp2, where)
+      if !ctx.usedBestEffortTasty then
+        expectValueTypeOrWildcard(tp1, where)
+        expectValueTypeOrWildcard(tp2, where)
       assertUnerased()
       unique(new CachedOrType(tp1, tp2, soft))
     }