Skip to content

Commit

Permalink
Limit prometheus exemplar labels (open-telemetry#6770)
Browse files Browse the repository at this point in the history
  • Loading branch information
@harshitrjpt
harshitrjpt committed Oct 15, 2024
1 parent e4d5109 commit 2bd6126
Show file tree
Hide file tree
Showing 2 changed files with 162 additions and 13 deletions.
42 changes: 29 additions & 13 deletions ...ometheus/src/main/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
import io.prometheus.metrics.model.snapshots.HistogramSnapshot.HistogramDataPointSnapshot;
import io.prometheus.metrics.model.snapshots.InfoSnapshot;
import io.prometheus.metrics.model.snapshots.InfoSnapshot.InfoDataPointSnapshot;
import io.prometheus.metrics.model.snapshots.Label;
import io.prometheus.metrics.model.snapshots.Labels;
import io.prometheus.metrics.model.snapshots.MetricMetadata;
import io.prometheus.metrics.model.snapshots.MetricSnapshot;
Expand All @@ -51,6 +52,7 @@
import io.prometheus.metrics.model.snapshots.SummarySnapshot;
import io.prometheus.metrics.model.snapshots.SummarySnapshot.SummaryDataPointSnapshot;
import io.prometheus.metrics.model.snapshots.Unit;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
Expand All @@ -77,6 +79,7 @@ final class Otel2PrometheusConverter {
private static final String OTEL_SCOPE_VERSION = "otel_scope_version";
private static final long NANOS_PER_MILLISECOND = TimeUnit.MILLISECONDS.toNanos(1);
static final int MAX_CACHE_SIZE = 10;
static final int EXEMPLAR_MAX_RUNES = 128;

private final boolean otelScopeEnabled;
@Nullable private final Predicate<String> allowedResourceAttributesFilter;
Expand Down Expand Up @@ -400,29 +403,42 @@ private Exemplars convertDoubleExemplars(List<DoubleExemplarData> exemplars) {
return Exemplars.of(result);
}

@Nullable
private Exemplar convertExemplar(double value, ExemplarData exemplar) {
SpanContext spanContext = exemplar.getSpanContext();
Labels labels = Labels.EMPTY;
if (spanContext.isValid()) {
return new Exemplar(
value,
labels =
convertAttributes(
null, // resource attributes are only copied for point's attributes
null, // scope attributes are only needed for point's attributes
null,
null,
exemplar.getFilteredAttributes(),
"trace_id",
spanContext.getTraceId(),
"span_id",
spanContext.getSpanId()),
exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
spanContext.getSpanId());
} else {
return new Exemplar(
value,
convertAttributes(
null, // resource attributes are only copied for point's attributes
null, // scope attributes are only needed for point's attributes
exemplar.getFilteredAttributes()),
exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
labels = convertAttributes(null, null, exemplar.getFilteredAttributes());
}
int runes = getRunes(labels);
if (runes > EXEMPLAR_MAX_RUNES) {
THROTTLING_LOGGER.log(
Level.WARNING,
"exemplar labels have " + runes + " runes, exceeding the limit of " + EXEMPLAR_MAX_RUNES);
return null;
}
return new Exemplar(value, labels, exemplar.getEpochNanos() / NANOS_PER_MILLISECOND);
}

private static int getRunes(Labels labels) {
int runes = 0;
for (Label l : labels) {
runes +=
new String(l.getName().getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8).length()
+ new String(l.getValue().getBytes(StandardCharsets.UTF_8), StandardCharsets.UTF_8)
.length();
}
return runes;
}

private InfoSnapshot makeTargetInfo(Resource resource) {
Expand Down
133 changes: 133 additions & 0 deletions ...heus/src/test/java/io/opentelemetry/exporter/prometheus/Otel2PrometheusConverterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
import static org.assertj.core.api.Assertions.assertThatCode;

import io.opentelemetry.api.common.Attributes;
import io.opentelemetry.api.trace.SpanContext;
import io.opentelemetry.api.trace.TraceFlags;
import io.opentelemetry.api.trace.TraceState;
import io.opentelemetry.sdk.common.InstrumentationScopeInfo;
import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
import io.opentelemetry.sdk.metrics.data.MetricData;
Expand All @@ -21,6 +24,7 @@
import io.opentelemetry.sdk.metrics.internal.data.ImmutableGaugeData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableHistogramData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableHistogramPointData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableLongExemplarData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableLongPointData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableMetricData;
import io.opentelemetry.sdk.metrics.internal.data.ImmutableSumData;
Expand Down Expand Up @@ -422,6 +426,42 @@ static MetricData createSampleMetricData(
throw new IllegalArgumentException("Unsupported metric data type: " + metricDataType);
}

static MetricData createLongMetricDataWithExemplar(
String metricName,
String metricUnit,
@Nullable Attributes attributes,
@Nullable Resource resource,
Attributes exemplarFilteredAttributes) {
Attributes attributesToUse = attributes == null ? Attributes.empty() : attributes;
Resource resourceToUse = resource == null ? Resource.getDefault() : resource;

return ImmutableMetricData.createLongSum(
resourceToUse,
InstrumentationScopeInfo.create("scope"),
metricName,
"description",
metricUnit,
ImmutableSumData.create(
true,
AggregationTemporality.CUMULATIVE,
Collections.singletonList(
ImmutableLongPointData.create(
0,
100000,
attributesToUse,
1L,
Collections.singletonList(
ImmutableLongExemplarData.create(
exemplarFilteredAttributes,
1L,
SpanContext.create(
"0669315b30dbe08683c19ed9bd24068b",
"049178b29912fdb4",
TraceFlags.getDefault(),
TraceState.getDefault()),
2))))));
}

@Test
void validateCacheIsBounded() {
AtomicInteger predicateCalledCount = new AtomicInteger();
Expand Down Expand Up @@ -478,4 +518,97 @@ void validateCacheIsBounded() {
// it never saw those resources before.
assertThat(predicateCalledCount.get()).isEqualTo(2);
}

@Test
void exemplarLabelsWithinLimit() throws IOException {

Otel2PrometheusConverter converter = new Otel2PrometheusConverter(true, null);
Attributes exemplarfilteredAttributes =
Attributes.of(
stringKey("client_address"),
"127.0.0.6",
stringKey("network_peer_address"),
"127.0.0.6");

MetricData metricDataWithExemplar =
createLongMetricDataWithExemplar(
"metric_hertz",
"hertz",
Attributes.of(stringKey("foo1"), "bar1", stringKey("foo2"), "bar2"),
Resource.create(
Attributes.of(stringKey("host"), "localhost", stringKey("cluster"), "mycluster")),
exemplarfilteredAttributes);
String expectedExemplarLabels =
"client_address=\"127.0.0.6\""
+ ",network_peer_address=\"127.0.0.6\",span_id=\"049178b29912fdb4\""
+ ",trace_id=\"0669315b30dbe08683c19ed9bd24068b\"";
ByteArrayOutputStream out = new ByteArrayOutputStream();
MetricSnapshots snapshots =
converter.convert(Collections.singletonList(metricDataWithExemplar));
ExpositionFormats.init().getOpenMetricsTextFormatWriter().write(out, snapshots);
String expositionFormat = new String(out.toByteArray(), StandardCharsets.UTF_8);

// extract the only metric line
List<String> metricLines =
Arrays.stream(expositionFormat.split("\n"))
.filter(line -> line.startsWith("metric_hertz"))
.collect(Collectors.toList());
assertThat(metricLines).hasSize(1);

// metric_hertz_total{foo1="bar1",foo2="bar2",otel_scope_name="scope"} 1.0 #
// {client_address="127.0.0.6",network_peer_address="127.0.0.6",span_id="0002",trace_id="0001"}
// 2.0
String metricLine = metricLines.get(0);
String exemplarPart = metricLine.substring(metricLine.indexOf("#") + 2);

String exemplarLabels =
exemplarPart.substring(exemplarPart.indexOf("{") + 1, exemplarPart.indexOf("}"));
assertThat(exemplarLabels).isEqualTo(expectedExemplarLabels);
}

@Test
void exemplarLabelsAboveLimit() throws IOException {

Otel2PrometheusConverter converter = new Otel2PrometheusConverter(true, null);
Attributes exemplarfilteredAttributes =
Attributes.of(
stringKey("client_address"),
"127.0.0.6",
stringKey("network_peer_address"),
"127.0.0.6",
stringKey("network_peer_port"),
"55579",
stringKey("server_address"),
"10.3.17.168",
stringKey("server_port"),
"8081",
stringKey("url_path"),
"/foo/bar");
MetricData metricDataWithExemplar =
createLongMetricDataWithExemplar(
"metric_hertz",
"hertz",
Attributes.of(stringKey("foo1"), "bar1", stringKey("foo2"), "bar2"),
Resource.create(
Attributes.of(stringKey("host"), "localhost", stringKey("cluster"), "mycluster")),
exemplarfilteredAttributes);
ByteArrayOutputStream out = new ByteArrayOutputStream();
MetricSnapshots snapshots =
converter.convert(Collections.singletonList(metricDataWithExemplar));
ExpositionFormats.init().getOpenMetricsTextFormatWriter().write(out, snapshots);
String expositionFormat = new String(out.toByteArray(), StandardCharsets.UTF_8);

// extract the only metric line
List<String> metricLines =
Arrays.stream(expositionFormat.split("\n"))
.filter(line -> line.startsWith("metric_hertz"))
.collect(Collectors.toList());
assertThat(metricLines).hasSize(1);

// metric_hertz_total{foo1="bar1",foo2="bar2",otel_scope_name="scope"} 1.0
// no exemplar data as runes limit was reached
String metricLine = metricLines.get(0);
int exemplarDelimitterPos = metricLine.indexOf("#");
assertThat(exemplarDelimitterPos).isEqualTo(-1);
}
}

0 comments on commit 2bd6126

Please sign in to comment.