diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/AnalysisScenario.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/AnalysisScenario.scala index abc2d4e21a..e6332ca008 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/AnalysisScenario.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/AnalysisScenario.scala @@ -6,6 +6,7 @@ import com.typesafe.config.Config import org.opalj.fpcf.AnalysisScenario.AnalysisAutoConfigKey import org.opalj.fpcf.AnalysisScenario.AnalysisSchedulingStrategyKey +import org.opalj.fpcf.scheduling.CleanupSpec import org.opalj.fpcf.scheduling.SchedulingStrategy import org.opalj.graphs.Graph import org.opalj.log.LogContext @@ -209,7 +210,8 @@ class AnalysisScenario[A](val ps: PropertyStore) { */ def computeSchedule( propertyStore: PropertyStore, - defaultAnalysis: PropertyBounds => Option[ComputationSpecification[A]] = _ => None + defaultAnalysis: PropertyBounds => Option[ComputationSpecification[A]] = _ => None, + cleanupSpec: Option[CleanupSpec] = None )( implicit logContext: LogContext ): Schedule[A] = { @@ -282,7 +284,8 @@ class AnalysisScenario[A](val ps: PropertyStore) { Schedule( scheduledBatches, - initializationData + initializationData, + cleanupSpec ) } } diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/FPCFAnalysesManager.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/FPCFAnalysesManager.scala index f0a7bccaaa..cd30cf0e6f 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/FPCFAnalysesManager.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/FPCFAnalysesManager.scala @@ -4,6 +4,7 @@ package fpcf import com.typesafe.config.Config +import org.opalj.fpcf.scheduling.CleanupSpec import org.opalj.log.LogContext import org.opalj.log.OPALLogger.debug import org.opalj.si.Project @@ -33,20 +34,22 @@ class FPCFAnalysesManager private[fpcf] (val project: Project) { def executedSchedules: List[Schedule[FPCFAnalysis]] = schedules final def runAll[T <: FPCFAnalysis]( - analyses: ComputationSpecification[T]* + cleanupSpec: CleanupSpec, + analyses: ComputationSpecification[T]* ): (PropertyStore, List[(ComputationSpecification[FPCFAnalysis], FPCFAnalysis)]) = { - runAll(analyses.to(Iterable)) + runAll(analyses.to(Iterable), cleanupSpec = Some(cleanupSpec)) } final def runAll[T <: FPCFAnalysis]( analyses: Iterable[ComputationSpecification[T]], - afterPhaseScheduling: List[ComputationSpecification[FPCFAnalysis]] => Unit = _ => () + afterPhaseScheduling: List[ComputationSpecification[FPCFAnalysis]] => Unit = _ => (), + cleanupSpec: Option[CleanupSpec] = None ): (PropertyStore, List[(ComputationSpecification[FPCFAnalysis], FPCFAnalysis)]) = this.synchronized { val scenario = AnalysisScenario(analyses.asInstanceOf[Iterable[ComputationSpecification[FPCFAnalysis]]], propertyStore) - val schedule = scenario.computeSchedule(propertyStore, FPCFAnalysesRegistry.defaultAnalysis) + val schedule = scenario.computeSchedule(propertyStore, FPCFAnalysesRegistry.defaultAnalysis, cleanupSpec) schedules ::= schedule if (trace) { debug("analysis progress", "executing " + schedule) } diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/PhaseConfiguration.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/PhaseConfiguration.scala index 62a0a2d3da..08ee5c98ca 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/PhaseConfiguration.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/PhaseConfiguration.scala @@ -4,5 +4,6 @@ package fpcf case class PhaseConfiguration[A]( propertyKinds: PropertyKindsConfiguration, - scheduled: List[ComputationSpecification[A]] + scheduled: List[ComputationSpecification[A]], + toDelete: Set[Int] = Set.empty ) diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyKey.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyKey.scala index 30b0f2e8a6..c1dd931186 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyKey.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyKey.scala @@ -123,6 +123,18 @@ object PropertyKey { */ def name(id: Int): String = propertyKeyNames(id) + def idByName(name: String): Int = + getByName(name).id + + /** + * Returns a [[SomePropertyKey]] associated with the given name. To get it by id use [[key]]. Throws an [[IllegalArgumentException]] if no [[PropertyKey]] exists for the given name. + */ + def getByName(name: String): SomePropertyKey = { + propertyKeys.find(k => PropertyKey.name(k.id) == name).getOrElse( + throw new IllegalArgumentException(s"Unknown property name: $name") + ) + } + final def name(pKind: PropertyKind): String = name(pKind.id) final def name(eOptionP: SomeEOptionP): String = name(eOptionP.pk.id) diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStore.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStore.scala index 7d7b4b2453..87267ed218 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStore.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStore.scala @@ -135,7 +135,8 @@ abstract class PropertyStore { // // - private val externalInformation = new ConcurrentHashMap[AnyRef, AnyRef]() + private[this] val externalInformation = new ConcurrentHashMap[AnyRef, AnyRef]() + protected[this] var currentPhaseDeletionMask: Array[Boolean] = Array.fill(PropertyKey.maxId + 1)(false) /** * Attaches or returns some information associated with the property store using a key object. @@ -500,6 +501,26 @@ abstract class PropertyStore { protected def doSet(e: Entity, p: Property): Unit + /** + * Removes [[PropertyKind]] from a [[PropertyStore]] using a pre-calculated [[currentPhaseDeletionMask]]. Calls [[clearPK]] which need to be implemented in the implementations of the [[PropertyStore]]. + */ + protected[fpcf] final def clearObsoletePropertyKinds(): Unit = { + val mask = currentPhaseDeletionMask + var index = 0 + while (index < mask.length) { + if (mask(index)) { + clearPK(index) + } + index += 1 + } + } + + /** + * Placeholder to remove a given [[PropertyKey]] from a [[PropertyStore]]. Needs to be overriden in the implementation for access to the given [[PropertyStore]]. + * @param id ID of the [[PropertyKey]] to be removed + */ + protected def clearPK(id: Int): Unit + /** * Associates the given entity with the newly computed intermediate property P. * @@ -539,7 +560,17 @@ abstract class PropertyStore { ) } - protected var subPhaseId: Int = 0 + final def setupPhase(configuration: PropertyKindsConfiguration, toDelete: Set[Int]): Unit = { + setupPhase( + configuration.propertyKindsComputedInThisPhase, + configuration.propertyKindsComputedInLaterPhase, + configuration.suppressInterimUpdates, + configuration.collaborativelyComputedPropertyKindsFinalizationOrder, + toDelete + ) + } + + protected[this] var subPhaseId: Int = 0 protected var hasSuppressedNotifications: Boolean = false @@ -571,7 +602,8 @@ abstract class PropertyStore { propertyKindsComputedInThisPhase: Set[PropertyKind], propertyKindsComputedInLaterPhase: Set[PropertyKind] = Set.empty, suppressInterimUpdates: Map[PropertyKind, Set[PropertyKind]] = Map.empty, - finalizationOrder: List[List[PropertyKind]] = List.empty + finalizationOrder: List[List[PropertyKind]] = List.empty, + toDelete: Set[Int] = Set.empty ): Unit = handleExceptions { if (!isIdle) { throw new IllegalStateException("computations are already running"); @@ -639,14 +671,23 @@ abstract class PropertyStore { hasSuppressedNotifications = suppressInterimUpdates.nonEmpty // Step 5 + // Set up a new deletionMask by resetting it first, then filling it with the help of toDelete. + val mask = currentPhaseDeletionMask + java.util.Arrays.fill(mask, false) + + toDelete.foreach(id => mask(id) = true) + + // Step 6 // Call `newPhaseInitialized` to enable subclasses to perform custom initialization steps // when a phase was set up. newPhaseInitialized( propertyKindsComputedInThisPhase, propertyKindsComputedInLaterPhase, suppressInterimUpdates, - finalizationOrder + finalizationOrder, + currentPhaseDeletionMask ) + } /** @@ -657,7 +698,8 @@ abstract class PropertyStore { propertyKindsComputedInThisPhase: Set[PropertyKind], propertyKindsComputedInLaterPhase: Set[PropertyKind], suppressInterimUpdates: Map[PropertyKind, Set[PropertyKind]], - finalizationOrder: List[List[PropertyKind]] + finalizationOrder: List[List[PropertyKind]], + phaseDeletionMask: Array[Boolean] ): Unit = { /*nothing to do*/ } /** diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStoreBasedCommandLineConfig.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStoreBasedCommandLineConfig.scala index a23686d53e..4d474677ac 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStoreBasedCommandLineConfig.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/PropertyStoreBasedCommandLineConfig.scala @@ -3,6 +3,7 @@ package org.opalj package fpcf import com.typesafe.config.Config +import com.typesafe.config.ConfigValueFactory import org.rogach.scallop.ScallopConf import org.rogach.scallop.flagConverter @@ -12,6 +13,7 @@ import org.opalj.cli.Arg import org.opalj.cli.ConvertedArg import org.opalj.cli.ForwardingArg import org.opalj.cli.OPALCommandLineConfig +import org.opalj.cli.ParsedArg import org.opalj.cli.PlainArg import org.opalj.fpcf.par.SchedulingStrategyArg import org.opalj.log.LogContext @@ -68,7 +70,10 @@ trait PropertyStoreBasedCommandLineConfig extends OPALCommandLineConfig { self: generalArgs( PropertyStoreThreadsNumArg, PropertyStoreDebugArg, - SchedulingStrategyArg + SchedulingStrategyArg, + DisableCleanupArg, + KeepPropertyKeysArg, + ClearPropertyKeysArg ) def setupPropertyStore(project: Project): (PropertyStore, Seconds) = { @@ -93,3 +98,49 @@ trait PropertyStoreBasedCommandLineConfig extends OPALCommandLineConfig { self: else FPCFAnalysesRegistry.lazyFactory(analysisName) } } + +object DisableCleanupArg extends PlainArg[Boolean] { + override val name: String = "disableCleanup" + override def description: String = "Disable cleanup of the PropertyStore inbetween phases" + override val defaultValue: Option[Boolean] = Some(false) + override def apply(config: Config, value: Option[Boolean]): Config = { + config.withValue( + "org.opalj.fpcf.AnalysisScenario.DisableCleanup", + ConfigValueFactory.fromAnyRef(value.getOrElse(false)) + ) + } +} + +object KeepPropertyKeysArg extends ParsedArg[List[String], List[SomePropertyKey]] { + override val name: String = "keepPropertyKeys" + override val description: String = "List of Properties to keep at the end of the analysis" + override val defaultValue: Option[List[String]] = None + + override def apply(config: Config, value: Option[List[SomePropertyKey]]): Config = { + config.withValue( + "org.opalj.fpcf.AnalysisScenario.KeepPropertyKeys", + ConfigValueFactory.fromAnyRef(value.getOrElse("")) + ) + } + + override def parse(arg: List[String]): List[SomePropertyKey] = { + arg.flatMap(_.split(",")).map(PropertyKey.getByName) + } +} + +object ClearPropertyKeysArg extends ParsedArg[List[String], List[SomePropertyKey]] { + override val name: String = "clearPropertyKeys" + override val description: String = "List of Properties to keep at the end of the analysis" + override val defaultValue: Option[List[String]] = None + + override def apply(config: Config, value: Option[List[SomePropertyKey]]): Config = { + config.withValue( + "org.opalj.fpcf.AnalysisScenario.ClearPropertyKeys", + ConfigValueFactory.fromAnyRef(value.getOrElse("")) + ) + } + + override def parse(arg: List[String]): List[SomePropertyKey] = { + arg.flatMap(_.split(",")).map(PropertyKey.getByName) + } +} diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/Schedule.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/Schedule.scala index 97bd0b7696..3a53978f55 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/Schedule.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/Schedule.scala @@ -2,6 +2,7 @@ package org.opalj package fpcf +import org.opalj.fpcf.scheduling.CleanupSpec import org.opalj.log.LogContext import org.opalj.log.OPALLogger.info import org.opalj.util.PerformanceEvaluation.time @@ -18,7 +19,8 @@ import org.opalj.util.elidedAssert */ case class Schedule[A]( batches: List[PhaseConfiguration[A]], - initializationData: Map[ComputationSpecification[A], Any] + initializationData: Map[ComputationSpecification[A], Any], + cleanupSpec: Option[CleanupSpec] ) extends ( ( PropertyStore, @@ -44,16 +46,24 @@ case class Schedule[A]( ): List[(ComputationSpecification[A], A)] = { implicit val logContext: LogContext = ps.logContext + val phases = cleanupSpec.map(scheduling.Cleanup.withPerPhaseCleanup(batches, ps, _)).getOrElse(batches) + var allExecutedAnalyses: List[(ComputationSpecification[A], A)] = Nil - batches.iterator.zipWithIndex foreach { batchId => - val (PhaseConfiguration(configuration, css), id) = batchId + phases.iterator.zipWithIndex foreach { batchId => + val (phase, id) = batchId + val configuration = phase.propertyKinds + val css = phase.scheduled if (trace) { info("analysis progress", s"setting up analysis phase $id: $configuration") + if (phase.toDelete.nonEmpty) { + info("analysis progress", s"to be deleted after this phase: " + phase.toDelete.map(PropertyKey.name)) + } } time { - ps.setupPhase(configuration) + ps.setupPhase(configuration, phase.toDelete) + afterPhaseSetup(configuration) elidedAssert(ps.isIdle, "the property store is not idle after phase setup") @@ -88,8 +98,9 @@ case class Schedule[A]( ) } } + // ... we are done now; the computed properties will no longer be computed! - ps.setupPhase(Set.empty, Set.empty) + ps.setupPhase(Set.empty, propertyKindsComputedInLaterPhase = Set.empty) allExecutedAnalyses } diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/par/PKECPropertyStore.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/par/PKECPropertyStore.scala index 3d5b2f8b7a..c46ea32317 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/par/PKECPropertyStore.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/par/PKECPropertyStore.scala @@ -528,15 +528,16 @@ class PKECPropertyStore( startThreads(new PartialPropertiesFinalizerThread(_)) subPhaseId += 1 - ps(AnalysisKeyId).clear() } + clearObsoletePropertyKinds() idle = true } - private val interimStates: Array[ArrayBuffer[EPKState]] = - Array.fill(THREAD_COUNT)(null) + override protected def clearPK(id: Int): Unit = ps(id).clear() + + private[this] val interimStates: Array[ArrayBuffer[EPKState]] = private val successors: Array[EPKState => Iterable[EPKState]] = Array.fill(THREAD_COUNT)(null) diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/Cleanup.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/Cleanup.scala new file mode 100644 index 0000000000..61b662391c --- /dev/null +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/Cleanup.scala @@ -0,0 +1,98 @@ +/* BSD 2-Clause License - see OPAL/LICENSE for details. */ +package org.opalj +package fpcf +package scheduling + +import scala.annotation.targetName + +/** + * Class that allows to configure the cleanup of the PropertyStore inbetween phases programmatically + * + * @param keep IDs of the PropertyKeys to be kept at the end + * @param clear IDs of the PropertyKeys to be definitely removed + * @param enable Allows the cleanup to be enabled since it is off by default + */ +final case class CleanupSpec( + keep: Set[Int] = Set.empty, + clear: Set[Int] = Set.empty, + enable: Boolean = false +) + +/** + * Companion Object + */ +object CleanupSpec { + /** + * Creates a [[CleanupSpec]] from given [[PropertyKey]]-names to keep and/or to clear. Also allows enabling the cleanup by setting 'enable' to 'true'. + * + * @param keep Names of PropertyKeys to be kept after the analyses + * @param clear Names of PropertyKeys to be removed after the analyses + * @param enable Setting this to 'true' enables the cleanup inbetween the phases + * @return A new [[CleanupSpec]] + */ + def apply(keep: Set[String], clear: Set[String], enable: Boolean)(implicit di: DummyImplicit): CleanupSpec = { + val toKeep = keep.map(PropertyKey.idByName) + val toClear = clear.map(PropertyKey.idByName) + CleanupSpec(toKeep, toClear, enable) + } + +} + +/** + * Handles calculation of per-phase cleanup + */ +object Cleanup { + + /** + * Calculates the properties to be safely removed inbetween phases. Returns an unmodified schedule if cleanup is disabled + */ + def withPerPhaseCleanup[A]( + schedule: List[PhaseConfiguration[A]], + ps: PropertyStore, + spec: CleanupSpec + ): List[PhaseConfiguration[A]] = { + // in case the cleanup is disabled, we don't want to modify the schedules per-phase "toDelete" + if (!spec.enable) return schedule + + // calculate the pks which are produced in any phase (so we can ignore the rest) + val producedInAnyPhase: Set[Int] = + schedule.iterator.flatMap(_.propertyKinds.propertyKindsComputedInThisPhase.map(_.id)).toSet + val neededLater = Array.fill[Set[Int]](schedule.size + 1)(Set.empty) + + var index = schedule.size - 1 + var usedInAnyPhase: Set[Int] = Set.empty + + // calculate neededLater and usedInAnyPhase iteratively while keeping neededLater minimal at each time + // + while (index >= 0) { + val usedInThisPhase: Set[Int] = + schedule(index).scheduled.iterator.flatMap(_.uses(ps).iterator).map(_.pk.id).toSet + neededLater(index) = usedInAnyPhase + usedInAnyPhase = usedInAnyPhase union usedInThisPhase + index -= 1 + } + + // store to keep track of already deleted pks to not delete them again in later phases + var alreadyCleared = Set.empty[Int] + val lastIndex = schedule.size - 1 + + // calculate the pks to be deleted after each phase + schedule.indices.iterator.map { index => + val producedHere = schedule(index) + + // only delete the pks the user wants to delete as it's the final phase. + // Anything not deleted will be returned + val toDelete = if (index == lastIndex) { + spec.clear -- alreadyCleared + } else { + // delete any pk safe to delete (not needed later) which isn't already deleted + // we also don't want to delete anything the user wants to keep + (((producedInAnyPhase -- neededLater(index)) -- alreadyCleared) -- spec.keep) union spec.clear + } + // keep track of the deleted pks + alreadyCleared ++= toDelete + producedHere.copy(toDelete = toDelete) + }.toList + } + +} diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/MaximumPhaseScheduling.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/MaximumPhaseScheduling.scala index d494219a41..3e51d47dfe 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/MaximumPhaseScheduling.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/scheduling/MaximumPhaseScheduling.scala @@ -97,7 +97,6 @@ abstract class MultiplePhaseScheduling extends SchedulingStrategy { remainingAnalyses --= phaseAnalyses computePhase(ps, phaseAnalyses, remainingAnalyses) } - schedule } @@ -168,3 +167,9 @@ object MaximumPhaseScheduling extends MultiplePhaseScheduling { (initialPhaseDependencyGraph.toMap, initialPhaseIndexToAnalyses.toMap) } } + +object CleanupCalculation { + final val PropertiesToKeepKey = s"${ConfigKeyPrefix}KeepPropertyKeys" + final val PropertiesToRemoveKey = s"${ConfigKeyPrefix}ClearPropertyKeys" + final val DisableCleanupKey = s"${ConfigKeyPrefix}DisableCleanup" +} diff --git a/OPAL/si/src/main/scala/org/opalj/fpcf/seq/PKESequentialPropertyStore.scala b/OPAL/si/src/main/scala/org/opalj/fpcf/seq/PKESequentialPropertyStore.scala index f4cd392262..f308e8dd9d 100644 --- a/OPAL/si/src/main/scala/org/opalj/fpcf/seq/PKESequentialPropertyStore.scala +++ b/OPAL/si/src/main/scala/org/opalj/fpcf/seq/PKESequentialPropertyStore.scala @@ -832,11 +832,14 @@ final class PKESequentialPropertyStore protected ( continueComputation } do () + clearObsoletePropertyKinds() idle = true if (exception != null) throw exception; } + override protected def clearPK(id: Int): Unit = ps(id).clear() + def shutdown(): Unit = {} }