twitter-server: Emit labels and the type of name in the metrics schem…

…a endpoint Problem We want to include label information and whether the metric in question is valid for dimensional export. Solution Export the info. JIRA Issues: CSL-11809 Differential Revision: https://phabricator.twitter.biz/D864694
twitter · Apr 13, 2022 · 7c107d8 · 7c107d8
1 parent 79fc647
commit 7c107d8
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 27 deletions.
diff --git a/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala b/server/src/main/scala/com/twitter/server/handler/MetricMetadataQueryHandler.scala
@@ -1,11 +1,17 @@
 package com.twitter.server.handler
 
 import com.twitter.finagle.Service
-import com.twitter.finagle.http.{MediaType, Request, Response, Uri}
-import com.twitter.finagle.stats.{StatsFormatter, metadataScopeSeparator}
+import com.twitter.finagle.http.MediaType
+import com.twitter.finagle.http.Request
+import com.twitter.finagle.http.Response
+import com.twitter.finagle.http.Uri
+import com.twitter.finagle.stats.MetricBuilder
+import com.twitter.finagle.stats.StatsFormatter
+import com.twitter.finagle.stats.metadataScopeSeparator
 import com.twitter.io.Buf
 import com.twitter.server.util.HttpUtils.newResponse
-import com.twitter.server.util.{AdminJsonConverter, MetricSchemaSource}
+import com.twitter.server.util.AdminJsonConverter
+import com.twitter.server.util.MetricSchemaSource
 import com.twitter.util.Future
 
 /**
@@ -86,8 +92,8 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo
 
   private[this] val statsFormatter = StatsFormatter.default
 
-  private[this] def query(keys: Iterable[String]) = {
-    keys.flatMap(k => {
+  private[this] def query(keys: Iterable[String]): Iterable[MetricBuilder] = {
+    keys.flatMap { k =>
       // histogram metric.
       if (k.lastIndexOf(statsFormatter.histogramSeparator) > k.lastIndexOf(
           metadataScopeSeparator())) {
@@ -101,7 +107,7 @@ class MetricMetadataQueryHandler(source: MetricSchemaSource = new MetricSchemaSo
       } else {
         source.getSchema(k)
       }
-    })
+    }
   }
 
   def apply(req: Request): Future[Response] = {

diff --git a/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala b/server/src/main/scala/com/twitter/server/util/MetricSchemaSerializer.scala
@@ -5,13 +5,42 @@ import com.fasterxml.jackson.databind.SerializerProvider
 import com.fasterxml.jackson.databind.ser.std.StdSerializer
 import com.twitter.finagle.stats.MetricBuilder.HistogramType
 import com.twitter.finagle.stats.MetricBuilder
+import com.twitter.finagle.stats.MetricBuilder.Identity
 import com.twitter.finagle.stats.StatsFormatter
 import com.twitter.finagle.stats.metadataScopeSeparator
 
 object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuilder]) {
 
   private[this] val statsFormatter = StatsFormatter.default
 
+  private[this] def dimensionallyValid(identity: Identity): Boolean = identity match {
+    case Identity.Full(_, _) => true
+    case Identity.Hierarchical(_, _) => false
+  }
+
+  private[this] def writeArray(
+    jsonGenerator: JsonGenerator,
+    name: String,
+    strings: Iterator[String]
+  ): Unit = {
+    jsonGenerator.writeArrayFieldStart(name)
+    strings.foreach(s => jsonGenerator.writeString(convertNullToString(s)))
+    jsonGenerator.writeEndArray()
+  }
+
+  private[this] def writeDictionary(
+    jsonGenerator: JsonGenerator,
+    name: String,
+    entries: Iterable[(String, String)]
+  ): Unit = {
+    jsonGenerator.writeObjectFieldStart(name)
+    entries.foreach {
+      case (k, v) =>
+        jsonGenerator.writeStringField(k, v)
+    }
+    jsonGenerator.writeEndObject()
+  }
+
   /**
    * This custom serializer is used to convert MetricSchemas to JSON for the metric_metadata
    * endpoint.
@@ -32,14 +61,19 @@ object SchemaSerializer extends StdSerializer[MetricBuilder](classOf[MetricBuild
     val formattedName = metricBuilder.name.mkString(metadataScopeSeparator())
     jsonGenerator.writeStartObject()
     jsonGenerator.writeStringField("name", formattedName)
-    jsonGenerator.writeArrayFieldStart("relative_name")
-    if (metricBuilder.relativeName != Seq.empty) {
-      metricBuilder.relativeName.foreach(segment =>
-        jsonGenerator.writeString(convertNullToString(segment)))
-    } else {
-      metricBuilder.name.foreach(segment => jsonGenerator.writeString(convertNullToString(segment)))
-    }
-    jsonGenerator.writeEndArray()
+
+    writeArray(
+      jsonGenerator,
+      "relative_name",
+      (if (metricBuilder.relativeName.nonEmpty) metricBuilder.relativeName
+       else metricBuilder.name).iterator)
+
+    writeDictionary(jsonGenerator, "labels", metricBuilder.identity.labels)
+
+    jsonGenerator.writeBooleanField(
+      "dimensional_support",
+      dimensionallyValid(metricBuilder.identity))
+
     val dataType = metricBuilder.metricType.toJsonString
     jsonGenerator.writeStringField("kind", dataType)
     jsonGenerator.writeObjectFieldStart("source")

diff --git a/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala b/server/src/test/scala/com/twitter/server/handler/MetricMetadataQueryHandlerTest.scala
@@ -1,11 +1,16 @@
 package com.twitter.server.handler
 
 import com.twitter.finagle.http.Request
-import com.twitter.finagle.stats.MetricBuilder.{CounterType, GaugeType, HistogramType}
+import com.twitter.finagle.stats.MetricBuilder.CounterType
+import com.twitter.finagle.stats.MetricBuilder.GaugeType
+import com.twitter.finagle.stats.MetricBuilder.HistogramType
 import com.twitter.finagle.stats._
-import com.twitter.finagle.stats.exp.{ExpressionSchema, ExpressionSchemaKey}
-import com.twitter.server.util.{JsonUtils, MetricSchemaSource}
-import com.twitter.util.{Await, Duration}
+import com.twitter.finagle.stats.exp.ExpressionSchema
+import com.twitter.finagle.stats.exp.ExpressionSchemaKey
+import com.twitter.server.util.JsonUtils
+import com.twitter.server.util.MetricSchemaSource
+import com.twitter.util.Await
+import com.twitter.util.Duration
 import org.scalatest.funsuite.AnyFunSuite
 
 class MetricMetadataQueryHandlerTest extends AnyFunSuite {
@@ -24,7 +29,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
         percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999),
         metricType = CounterType,
         statsReceiver = null
-      ),
+      ).withLabels(Map("label1" -> "value1", "label2" -> "value2")).withDimensionalSupport,
     "your/fine/gauge" ->
       MetricBuilder(
         keyIndicator = false,
@@ -45,7 +50,7 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
         percentiles = IndexedSeq(0.5, 0.9, 0.95, 0.99, 0.999, 0.9999),
         metricType = HistogramType,
         statsReceiver = null
-      ),
+      ).withLabels(labels = Map("label1" -> "value1")).withDimensionalSupport,
     "my/bad/null/counter" ->
       MetricBuilder(
         keyIndicator = true,
@@ -111,6 +116,11 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "my/cool/counter",
       |      "relative_name" : ["my","cool","counter"],
+      |      "labels" : {
+      |        "label1" : "value1",
+      |        "label2" : "value2"
+      |      },
+      |      "dimensional_support" : true,
       |      "kind" : "counter",
       |      "source" : {
       |        "class": "finagle.stats.cool",
@@ -125,6 +135,8 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "your/fine/gauge",
       |      "relative_name" : ["your","fine","gauge"],
+      |      "labels" : {},
+      |      "dimensional_support" : false,
       |      "kind" : "gauge",
       |      "source" : {
       |        "class": "finagle.stats.your",
@@ -139,6 +151,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "my/only/histo",
       |      "relative_name" : ["my","only","histo"],
+      |      "labels" : {
+      |        "label1" : "value1"
+      |      },
+      |      "dimensional_support" : true,
       |      "kind" : "histogram",
       |      "source" : {
       |        "class": "Unspecified",
@@ -166,6 +182,8 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "my/bad/null/counter",
       |      "relative_name" : ["my","bad", "null", "counter"],
+      |      "labels" : {},
+      |      "dimensional_support" : false,
       |      "kind" : "counter",
       |      "source" : {
       |        "class": "finagle.stats.bad",
@@ -187,6 +205,8 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "your/fine/gauge",
       |      "relative_name" : ["your","fine","gauge"],
+      |      "labels" : {},
+      |      "dimensional_support" : false,
       |      "kind" : "gauge",
       |      "source" : {
       |        "class": "finagle.stats.your",
@@ -207,6 +227,11 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "my/cool/counter",
       |      "relative_name" : ["my","cool","counter"],
+      |      "labels" : {
+      |        "label1" : "value1",
+      |        "label2" : "value2"
+      |      },
+      |      "dimensional_support" : true,
       |      "kind" : "counter",
       |      "source" : {
       |        "class": "finagle.stats.cool",
@@ -221,6 +246,8 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "your/fine/gauge",
       |      "relative_name" : ["your","fine","gauge"],
+      |      "labels" : {},
+      |      "dimensional_support" : false,
       |      "kind" : "gauge",
       |      "source" : {
       |        "class": "finagle.stats.your",
@@ -242,6 +269,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "my/only/histo",
       |      "relative_name" : ["my","only","histo"],
+      |      "labels" : {
+      |        "label1" : "value1"
+      |      },
+      |      "dimensional_support" : true,
       |      "kind" : "histogram",
       |      "source" : {
       |        "class": "Unspecified",
@@ -275,6 +306,11 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "my/cool/counter",
       |      "relative_name" : ["my","cool","counter"],
+      |      "labels" : {
+      |        "label1" : "value1",
+      |        "label2" : "value2"
+      |      },
+      |      "dimensional_support" : true,
       |      "kind" : "counter",
       |      "source" : {
       |        "class": "finagle.stats.cool",
@@ -289,6 +325,10 @@ class MetricMetadataQueryHandlerTest extends AnyFunSuite {
       |     {
       |      "name" : "my/only/histo",
       |      "relative_name" : ["my","only","histo"],
+      |      "labels" : {
+      |        "label1" : "value1"
+      |      },
+      |      "dimensional_support" : true,
       |      "kind" : "histogram",
       |      "source" : {
       |        "class": "Unspecified",

diff --git a/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala b/server/src/test/scala/com/twitter/server/util/MetricSchemaJsonModuleTest.scala
@@ -1,13 +1,12 @@
 package com.twitter.server.util
 
 import com.fasterxml.jackson.databind.ObjectMapper
-import com.fasterxml.jackson.module.scala.{DefaultScalaModule, ScalaObjectMapper}
-import com.twitter.finagle.stats.MetricBuilder.{
-  CounterType,
-  CounterishGaugeType,
-  GaugeType,
-  HistogramType
-}
+import com.fasterxml.jackson.module.scala.DefaultScalaModule
+import com.fasterxml.jackson.module.scala.ScalaObjectMapper
+import com.twitter.finagle.stats.MetricBuilder.CounterType
+import com.twitter.finagle.stats.MetricBuilder.CounterishGaugeType
+import com.twitter.finagle.stats.MetricBuilder.GaugeType
+import com.twitter.finagle.stats.MetricBuilder.HistogramType
 import com.twitter.finagle.stats._
 import org.scalatest.funsuite.AnyFunSuite
 
@@ -53,6 +52,8 @@ class MetricSchemaJsonModuleTest extends AnyFunSuite {
     Set(
       "name",
       "relative_name",
+      "labels",
+      "dimensional_support",
       "kind",
       "source",
       "description",