Fix metrics sourcetype annotation

.chloggen/fixmetricssourcetypeannotation.yaml

@@ -0,0 +1,12 @@
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+# The name of the component, or a single word describing the area of concern, (e.g. agent, clusterReceiver, gateway, operator, chart, other)
+component: other
+# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Fixed updating metrics' sourcetype with annotations
+# One or more tracking issues related to the change
+issues: [1375]
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:

docs/advanced-configuration.md

@@ -564,7 +564,8 @@ Manage Splunk OTel Collector Logging with these supported annotations.
 * Filter logs using pod and/or namespace annotation
   * If `logsCollection.containers.useSplunkIncludeAnnotation` is `false` (default: false), set `splunk.com/exclude` annotation to `true` on pod and/or namespace to exclude its logs from ingested.
   * If `logsCollection.containers.useSplunkIncludeAnnotation` is `true` (default: false), set `splunk.com/include` annotation to `true` on pod and/or namespace to only include its logs from ingested. All other logs will be ignored.
-* Use `splunk.com/sourcetype` annotation on pod to overwrite `sourcetype` field. If not set, it is dynamically generated to be `kube:container:CONTAINER_NAME`.
+* For logs, use `splunk.com/sourcetype` annotation on pod to overwrite `sourcetype` field. If not set, it is dynamically generated to be `kube:container:CONTAINER_NAME`.
+* For metrics, use the `splunk.com/sourcetype` annotation on the namespace to override the sourcetype field. If not set, it defaults to `httpevent`.
 
 ### Performance of native OpenTelemetry logs collection
 

examples/disable-persistence-queue-traces/rendered_manifests/configmap-agent.yaml

@@ -165,8 +165,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/disable-persistence-queue-traces/rendered_manifests/configmap-cluster-receiver.yaml

@@ -50,8 +50,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/disable-persistence-queue-traces/rendered_manifests/daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         component: otel-collector-agent
         release: default
       annotations:
-        checksum/config: 8f13ff66652ba33fe2a9aee4c1a2e43517bfe12d57093be5d675ef515f537d78
+        checksum/config: 8cf566b711b895e8209d8c055fc25b2245e74abe84743cf64655a07cb78fee92
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/disable-persistence-queue-traces/rendered_manifests/deployment-cluster-receiver.yaml

@@ -31,7 +31,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 31057c8fa63ede3a8e48bce75a40865619cdd244a7cc16ade3471013072fb603
+        checksum/config: 130873286d49dc56b755805d556b3094fcb6696ce4e9326408389c58942d46db
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

examples/enable-persistence-queue/rendered_manifests/configmap-agent.yaml

@@ -165,8 +165,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/enable-persistence-queue/rendered_manifests/configmap-cluster-receiver.yaml

@@ -50,8 +50,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/enable-persistence-queue/rendered_manifests/daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         component: otel-collector-agent
         release: default
       annotations:
-        checksum/config: 7e28ba091fe865d050d7328d4484a25a517e3f7937d660ab8aca68b1f7206d21
+        checksum/config: 07fe1f32f371cae54b4ed702d65fd1a60a134f0c4d8b66dcfec1eb81ed1907bc
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/enable-persistence-queue/rendered_manifests/deployment-cluster-receiver.yaml

@@ -31,7 +31,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 31057c8fa63ede3a8e48bce75a40865619cdd244a7cc16ade3471013072fb603
+        checksum/config: 130873286d49dc56b755805d556b3094fcb6696ce4e9326408389c58942d46db
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

examples/multi-metrics/rendered_manifests/configmap-agent.yaml

@@ -137,8 +137,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/multi-metrics/rendered_manifests/configmap-cluster-receiver.yaml

@@ -50,8 +50,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/multi-metrics/rendered_manifests/daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         component: otel-collector-agent
         release: default
       annotations:
-        checksum/config: 5b4ddaf2cc99de360060d11ef3f956eaf152f249d2aeb62cf477602745b4cea1
+        checksum/config: 03163bd786efbdb0c43d07cb64eb17cf6cd9595af1c5db2ccf2c4c84615a2dfb
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/multi-metrics/rendered_manifests/deployment-cluster-receiver.yaml

@@ -31,7 +31,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 31057c8fa63ede3a8e48bce75a40865619cdd244a7cc16ade3471013072fb603
+        checksum/config: 130873286d49dc56b755805d556b3094fcb6696ce4e9326408389c58942d46db
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

examples/only-metrics-platform/rendered_manifests/configmap-agent.yaml

@@ -110,8 +110,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/only-metrics-platform/rendered_manifests/configmap-cluster-receiver.yaml

@@ -50,8 +50,12 @@ data:
       k8sattributes/metrics:
         extract:
           annotations:
+          - from: namespace
+            key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: pod
             key: splunk.com/sourcetype
+            tag_name: com.splunk.sourcetype
           - from: namespace
             key: splunk.com/metricsIndex
             tag_name: com.splunk.index

examples/only-metrics-platform/rendered_manifests/daemonset.yaml

@@ -32,7 +32,7 @@ spec:
         component: otel-collector-agent
         release: default
       annotations:
-        checksum/config: 1d48c12fc0681c6793c2c6e581e5199248d33a81165761431e28cc813cecd5c5
+        checksum/config: c3dbc1971b6dfeee15734ef1758d5b78e03f60fa6ce35f5d58b4df90ea71c24d
         kubectl.kubernetes.io/default-container: otel-collector
     spec:
       hostNetwork: true

examples/only-metrics-platform/rendered_manifests/deployment-cluster-receiver.yaml

@@ -31,7 +31,7 @@ spec:
         component: otel-k8s-cluster-receiver
         release: default
       annotations:
-        checksum/config: 31057c8fa63ede3a8e48bce75a40865619cdd244a7cc16ade3471013072fb603
+        checksum/config: 130873286d49dc56b755805d556b3094fcb6696ce4e9326408389c58942d46db
     spec:
       serviceAccountName: default-splunk-otel-collector
       nodeSelector:

helm-charts/splunk-otel-collector/templates/config/_common.tpl

@@ -232,6 +232,10 @@ k8sattributes/clusterReceiver:
     {{- if eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true"}}
     annotations:
       - key: splunk.com/sourcetype
+        tag_name: com.splunk.sourcetype
+        from: namespace
+      - key: splunk.com/sourcetype
+        tag_name: com.splunk.sourcetype
         from: pod
       - key: splunk.com/index
         tag_name: com.splunk.index
@@ -266,6 +270,10 @@ k8sattributes/metrics:
     metadata: []
     annotations:
       - key: splunk.com/sourcetype
+        tag_name: com.splunk.sourcetype
+        from: namespace
+      - key: splunk.com/sourcetype
+        tag_name: com.splunk.sourcetype
         from: pod
       - key: splunk.com/metricsIndex
         tag_name: com.splunk.index
@@ -338,6 +346,20 @@ resource/logs:
     {{- end }}
 {{- end }}
 
+{{/*
+Resource processor for metrics manipulations
+*/}}
+{{- define "splunk-otel-collector.resourceMetricsProcessor" -}}
+resource/metrics:
+  attributes:
+    {{- if .Values.splunkPlatform.sourcetype }}
+    # Insert the sourcetype value from values.yaml if it has not already been set through annotations.
+    - key: com.splunk.sourcetype
+      value: "{{.Values.splunkPlatform.sourcetype }}"
+      action: insert
+    {{- end }}
+{{- end }}
+
 {{/*
 The transform processor adds service.name attribute to logs the same way as it's done by istio for the generated traces
 https://github.com/istio/istio/blob/6237cb4e63cf9a332327cc0a815d6b46257e6f8a/pkg/config/analysis/analyzers/testdata/common/sidecar-injector-configmap.yaml#L110-L115
@@ -435,9 +457,6 @@ splunk_hec/platform_metrics:
   token: "${SPLUNK_PLATFORM_HEC_TOKEN}"
   index: {{ .Values.splunkPlatform.metricsIndex | quote }}
   source: {{ .Values.splunkPlatform.source | quote }}
-  {{- if .Values.splunkPlatform.sourcetype }}
-  sourcetype: {{ .Values.splunkPlatform.sourcetype | quote }}
-  {{- end }}
   max_idle_conns: {{ .Values.splunkPlatform.maxConnections }}
   max_idle_conns_per_host: {{ .Values.splunkPlatform.maxConnections }}
   disable_compression: {{ .Values.splunkPlatform.disableCompression }}

helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl

@@ -648,6 +648,9 @@ processors:
   {{- include "splunk-otel-collector.k8sAttributesSplunkPlatformMetrics" . | nindent 2 }}
     filter:
       node_from_env_var: K8S_NODE_NAME
+  {{- if .Values.splunkPlatform.sourcetype }}
+  {{- include "splunk-otel-collector.resourceMetricsProcessor" . | nindent 2 }}
+  {{- end }}
   {{- end }}
 
   {{- if eq .Values.logsEngine "fluentd" }}
@@ -1026,6 +1029,9 @@ service:
         {{- end }}
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if $gatewayEnabled }}
@@ -1052,6 +1058,9 @@ service:
         - resource
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }}

helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl

@@ -33,6 +33,9 @@ processors:
   {{- include "splunk-otel-collector.k8sAttributesSplunkPlatformMetrics" . | nindent 2 }}
     filter:
       node_from_env_var: K8S_NODE_NAME
+  {{- if .Values.splunkPlatform.sourcetype }}
+  {{- include "splunk-otel-collector.resourceMetricsProcessor" . | nindent 2 }}
+  {{- end }}
   {{- end }}
 
   {{- include "splunk-otel-collector.resourceLogsProcessor" . | nindent 2 }}
@@ -207,6 +210,9 @@ service:
         {{- end }}
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }}
@@ -261,6 +267,9 @@ service:
         {{- end }}
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }}

helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl

@@ -79,6 +79,9 @@ processors:
   {{- include "splunk-otel-collector.k8sAttributesSplunkPlatformMetrics" . | nindent 2 }}
     filter:
       node_from_env_var: K8S_NODE_NAME
+  {{- if .Values.splunkPlatform.sourcetype }}
+  {{- include "splunk-otel-collector.resourceMetricsProcessor" . | nindent 2 }}
+  {{- end }}
   {{- end }}
 
   batch:
@@ -237,6 +240,9 @@ service:
         - resource
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
         - resource/k8s_cluster
       exporters:
@@ -259,6 +265,9 @@ service:
         - resource
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }}
@@ -280,6 +289,9 @@ service:
         - resource
         {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }}
         - k8sattributes/metrics
+        {{- if .Values.splunkPlatform.sourcetype }}
+        - resource/metrics
+        {{- end }}
         {{- end }}
       exporters:
         {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }}

test/k8s_logging_tests/test_config_logging.py

@@ -81,3 +81,24 @@ def test_metric_index_from_annotations(setup, index, expected):
     logger.info("Splunk received %s events in the last minute",
                 len(events))
     assert len(events) >= expected
+
+@pytest.mark.parametrize("index,sourcetype,expected", [
+    ("test_metrics", "sourcetype-anno", 1)
+])
+def test_metric_sourcetype_from_annotations(setup, index, sourcetype, expected):
+
+    '''
+    Test that metrics are being assigned the "sourcetype-anno" sourcetype, as defined by splunk.com/sourcetype annotation added during setup
+    '''
+    logger.info("testing for metrics index={0} sourcetype={1} expected={2} event(s)".format(index, sourcetype, expected))
+    search_query = "index={0} filter=\"sourcetype={1}\"".format(index, sourcetype)
+
+    events = check_events_from_splunk(start_time="-1h@h",
+                                      url=setup["splunkd_url"],
+                                      user=setup["splunk_user"],
+                                      query=["mpreview {0}".format(
+                                          search_query)],
+                                      password=setup["splunk_password"])
+    logger.info("Splunk received %s events in the last minute",
+                len(events))
+    assert len(events) >= expected

test/test_setup.yaml

@@ -26,6 +26,7 @@ metadata:
   name: default
   annotations:
     splunk.com/metricsIndex: test_metrics
+    splunk.com/sourcetype: sourcetype-anno
 ---
 apiVersion: batch/v1
 kind: Job