finagle: more comprehensive support for dimensional metrics

==== Problem
Finagle doesn't have comprehensive support for dimensional metrics

=== Solution
1. Promote the visibility of `hierarchicalScope`, `dimensionalScope`, and `label` in `StatsReceiver` to public. This allows consumers to define their metric schemas for dual-consumption by the flat JSON & Prometheus exporters
2. Add the flag `com.twitter.finagle.stats.fullTranslation`. This allows service owners to express that raw `.scope` calls on `StatsReceiver` instances to propagate both hierarchical & dimensional metrics
3. Add the flag `com.twitter.finagle.stats.biasToFull`. This allows service owners to express that metrics with an indeterminate identity (have not been explicitly set as dimensional) to be exported through Prometheus anyways.
4. Remove the redundant dimensional scopes set on default Finagle client/service metrics
5. Remove the redundant `app` dimensional scope set on metrics exporter through `DefaultStatsReceiver`
6. Adjust, backwards-compatibly, the JVM metrics to utilize the new APIs and export the memory manager, pool, etc as a label instead of a hierarchical scope
7. Adjust, backwards-compatibly, some other internal use-cases to use labels instead of hierarchical scope

Differential Revision: https://phabricator.twitter.biz/D1218090

Author: keremnc

CHANGELOG.rst

@@ -11,6 +11,8 @@ New Features
 ~~~~~~~~~~~~
 
 * util-jvm: Add gc pause stats for all collector pools, including G1. ``PHAB_ID=D1176049``
+* util-stats: Expose dimensional metrics APIs and allow metrics with an indeterminate
+  identity to be exported through the Prometheus exporter. ``PHAB_ID=D1218090``
 
 
 24.5.0
@@ -21,6 +23,8 @@ Runtime Behavior Changes
 
 * util-app: When the application exits due to an error on startup, the error and
   and stack trace are printed to stderr in addition to the existing stdout. ``PHAB_ID=D1116753``
+* util-jvm: Memory manager & pool metrics are labelled dimensionally rather than hierarchically
+  when exported as dimensional metrics. ``PHAB_ID=D1218090``
 
 
 23.11.0

util-jvm/src/main/scala/com/twitter/jvm/Allocations.scala

@@ -41,7 +41,8 @@ private[jvm] class Allocations(statsReceiver: StatsReceiver) {
   private[this] val beanAndListeners =
     new LinkedBlockingQueue[(NotificationEmitter, NotificationListener)]()
 
-  private[this] val edenGcPauses = statsReceiver.scope("eden").stat("pause_msec")
+  private[this] val edenGcPauses = statsReceiver
+    .hierarchicalScope("eden").label("jmm", "eden").stat("pause_msec")
 
   private[jvm] def start(): Unit = {
     edenPool

util-jvm/src/main/scala/com/twitter/jvm/JvmStats.scala

@@ -113,15 +113,17 @@ object JvmStats {
     val postGCStats = memStats.scope("postGC")
     memPool.foreach { pool =>
       val name = pool.getName.regexSub("""[^\w]""".r) { m => "_" }
+      val currentPoolScope = currentMem.hierarchicalScope(name).label("pool", name)
+      val postGcPoolScope = postGCStats.hierarchicalScope(name).label("pool", name)
       if (pool.getCollectionUsage != null) {
         def usage = pool.getCollectionUsage // this is a snapshot, we can't reuse the value
-        gauges.add(postGCStats.addGauge(name, "used") { usage.getUsed.toFloat })
+        gauges.add(postGcPoolScope.addGauge("used") { usage.getUsed.toFloat })
       }
       if (pool.getUsage != null) {
         def usage = pool.getUsage // this is a snapshot, we can't reuse the value
-        val usageGauge = currentMem.addGauge(name, "used") { usage.getUsed.toFloat }
+        val usageGauge = currentPoolScope.addGauge("used") { usage.getUsed.toFloat }
         gauges.add(usageGauge)
-        gauges.add(currentMem.addGauge(name, "max") { usage.getMax.toFloat })
+        gauges.add(currentPoolScope.addGauge("max") { usage.getMax.toFloat })
 
         // register memory usage expression
         currentMem.registerExpression(
@@ -171,27 +173,29 @@ object JvmStats {
         val bufferPoolStats = memStats.scope("buffer")
         jBufferPool.asScala.foreach { bp =>
           val name = bp.getName
-          gauges.add(bufferPoolStats.addGauge(name, "count") { bp.getCount.toFloat })
-          gauges.add(bufferPoolStats.addGauge(name, "used") { bp.getMemoryUsed.toFloat })
-          gauges.add(bufferPoolStats.addGauge(name, "max") { bp.getTotalCapacity.toFloat })
+          val bufferPoolScope = bufferPoolStats.hierarchicalScope(name).label("pool", name)
+          gauges.add(bufferPoolScope.addGauge("count") { bp.getCount.toFloat })
+          gauges.add(bufferPoolScope.addGauge("used") { bp.getMemoryUsed.toFloat })
+          gauges.add(bufferPoolScope.addGauge("max") { bp.getTotalCapacity.toFloat })
         }
     }
 
     val gcPool = ManagementFactory.getGarbageCollectorMXBeans.asScala
     val gcStats = stats.scope("gc")
     gcPool.foreach { gc =>
       val name = gc.getName.regexSub("""[^\w]""".r) { m => "_" }
+      val poolScope = gcStats.hierarchicalScope(name).label("jmm", name)
       val poolCycles =
-        gcStats.addGauge(
-          gcStats.metricBuilder(GaugeType).withCounterishGauge.withName(name, "cycles")) {
+        poolScope.addGauge(
+          poolScope.metricBuilder(GaugeType).withCounterishGauge.withName("cycles")) {
           gc.getCollectionCount.toFloat
         }
-      val poolMsec = gcStats.addGauge(
-        gcStats.metricBuilder(GaugeType).withCounterishGauge.withName(name, "msec")) {
+      val poolMsec = poolScope.addGauge(
+        poolScope.metricBuilder(GaugeType).withCounterishGauge.withName("msec")) {
         gc.getCollectionTime.toFloat
       }
 
-      val gcPauseStat = gcStats.stat(name, "pause_msec")
+      val gcPauseStat = poolScope.stat("pause_msec")
       gc.asInstanceOf[NotificationEmitter].addNotificationListener(
           new NotificationListener {
             override def handleNotification(
@@ -244,7 +248,7 @@ object JvmStats {
     allocations.start()
     if (allocations.trackingEden) {
       val allocationStats = memStats.scope("allocations")
-      val eden = allocationStats.scope("eden")
+      val eden = allocationStats.hierarchicalScope("eden").label("space", "eden")
       gauges.add(eden.addGauge("bytes") { allocations.eden.toFloat })
     }
 

util-stats/src/main/scala/com/twitter/finagle/stats/LoadedStatsReceiver.scala

@@ -23,10 +23,7 @@ object LoadedStatsReceiver extends StatsReceiverProxy {
  * [[com.twitter.finagle.util.LoadService]] mechanism.
  */
 object DefaultStatsReceiver extends StatsReceiverProxy {
-  val self: StatsReceiver =
-    LoadedStatsReceiver
-      .dimensionalScope("app")
-      .label("implementation", "app")
+  val self: StatsReceiver = LoadedStatsReceiver.label("implementation", "app")
 
   override def repr: DefaultStatsReceiver.type = this
 

util-stats/src/main/scala/com/twitter/finagle/stats/MetricBuilder.scala

@@ -1,5 +1,6 @@
 package com.twitter.finagle.stats
 
+import com.twitter.app.GlobalFlag
 import com.twitter.finagle.stats.MetricBuilder.CounterType
 import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType
 import com.twitter.finagle.stats.MetricBuilder.GaugeType
@@ -10,6 +11,12 @@ import com.twitter.finagle.stats.MetricBuilder.MetricType
 import com.twitter.finagle.stats.MetricBuilder.UnlatchedCounter
 import scala.annotation.varargs
 
+object biasToFull
+    extends GlobalFlag[Boolean](
+      false,
+      "Bias metrics with an indeterminate identity to be exported through all available exporters"
+    )
+
 /**
  * Represents the "role" this service plays with respect to this metric.
  *
@@ -225,7 +232,7 @@ object MetricBuilder {
      * The current behavior is to default to [[IdentityType.HierarchicalOnly]]
      */
     private[twitter] def toResolvedIdentityType(identityType: IdentityType): ResolvedIdentityType =
-      identityType.bias(IdentityType.HierarchicalOnly)
+      identityType.bias(if (biasToFull()) IdentityType.Full else IdentityType.HierarchicalOnly)
 
     /** An [[IdentityType]] that cannot be [[NonDeterminate]] */
     sealed abstract class ResolvedIdentityType extends IdentityType

util-stats/src/main/scala/com/twitter/finagle/stats/StatsReceiver.scala

@@ -401,7 +401,7 @@ trait StatsReceiver {
    * Create a new `StatsReceiver` that will add a scope that is only used when the metric is
    * emitted in hierarchical form.
    */
-  private[finagle] final def hierarchicalScope(namespace: String): StatsReceiver = {
+  def hierarchicalScope(namespace: String): StatsReceiver = {
     if (namespace == "") this
     else
       new ScopeTranslatingStatsReceiver(
@@ -414,7 +414,7 @@ trait StatsReceiver {
    * Create a new `StatsReceiver` that will add a scope that is only used when the metric is
    * emitted in dimensional form.
    */
-  private[finagle] final def dimensionalScope(namespace: String): StatsReceiver = {
+  def dimensionalScope(namespace: String): StatsReceiver = {
     if (namespace == "") this
     else
       new ScopeTranslatingStatsReceiver(
@@ -426,7 +426,7 @@ trait StatsReceiver {
   /**
    * Create a new `StatsReceiver` that will add the specified label to all created metrics.
    */
-  private[finagle] final def label(labelName: String, labelValue: String): StatsReceiver = {
+  def label(labelName: String, labelValue: String): StatsReceiver = {
     require(labelName.nonEmpty)
     if (labelValue == "") this
     else new LabelTranslatingStatsReceiver(this, labelName, labelValue)

util-stats/src/test/scala/com/twitter/finagle/stats/DefaultStatsReceiverTest.scala

@@ -3,12 +3,12 @@ package com.twitter.finagle.stats
 import org.scalatest.funsuite.AnyFunSuite
 
 class DefaultStatsReceiverTest extends AnyFunSuite {
-  test("DefaultStatsReceiver has app in dimensional scope and label") {
+  test("DefaultStatsReceiver initializes dimensional scope and label") {
     LoadedStatsReceiver.self = new InMemoryStatsReceiver
     val fooCounter = DefaultStatsReceiver.counter("foo")
     val identity = fooCounter.metadata.toMetricBuilder.get.identity
 
-    assert(identity.dimensionalName == Seq("app", "foo"))
+    assert(identity.dimensionalName == Seq("foo"))
     assert(identity.hierarchicalName == Seq("foo"))
     assert(identity.labels == Map("implementation" -> "app"))
   }