diff --git a/.chloggen/fixmetricssourcetypeannotation.yaml b/.chloggen/fixmetricssourcetypeannotation.yaml new file mode 100644 index 0000000000..85a1c37fc6 --- /dev/null +++ b/.chloggen/fixmetricssourcetypeannotation.yaml @@ -0,0 +1,12 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix +# The name of the component, or a single word describing the area of concern, (e.g. agent, clusterReceiver, gateway, operator, chart, other) +component: other +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Fixed updating metrics' sourcetype with annotations +# One or more tracking issues related to the change +issues: [1375] +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/docs/advanced-configuration.md b/docs/advanced-configuration.md index ae30e35700..7c555f178e 100644 --- a/docs/advanced-configuration.md +++ b/docs/advanced-configuration.md @@ -564,7 +564,8 @@ Manage Splunk OTel Collector Logging with these supported annotations. * Filter logs using pod and/or namespace annotation * If `logsCollection.containers.useSplunkIncludeAnnotation` is `false` (default: false), set `splunk.com/exclude` annotation to `true` on pod and/or namespace to exclude its logs from ingested. * If `logsCollection.containers.useSplunkIncludeAnnotation` is `true` (default: false), set `splunk.com/include` annotation to `true` on pod and/or namespace to only include its logs from ingested. All other logs will be ignored. -* Use `splunk.com/sourcetype` annotation on pod to overwrite `sourcetype` field. If not set, it is dynamically generated to be `kube:container:CONTAINER_NAME`. +* For logs, use `splunk.com/sourcetype` annotation on pod to overwrite `sourcetype` field. If not set, it is dynamically generated to be `kube:container:CONTAINER_NAME`. +* For metrics, use the `splunk.com/sourcetype` annotation on the namespace to override the sourcetype field. If not set, it defaults to `httpevent`. ### Performance of native OpenTelemetry logs collection diff --git a/examples/disable-persistence-queue-traces/rendered_manifests/configmap-agent.yaml b/examples/disable-persistence-queue-traces/rendered_manifests/configmap-agent.yaml index 43afd2ee1f..57b31106d7 100644 --- a/examples/disable-persistence-queue-traces/rendered_manifests/configmap-agent.yaml +++ b/examples/disable-persistence-queue-traces/rendered_manifests/configmap-agent.yaml @@ -165,8 +165,12 @@ data: k8sattributes/metrics: extract: annotations: + - from: namespace + key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: pod key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: namespace key: splunk.com/metricsIndex tag_name: com.splunk.index diff --git a/examples/disable-persistence-queue-traces/rendered_manifests/configmap-cluster-receiver.yaml b/examples/disable-persistence-queue-traces/rendered_manifests/configmap-cluster-receiver.yaml index f8f09c8b36..7b89f46ffe 100644 --- a/examples/disable-persistence-queue-traces/rendered_manifests/configmap-cluster-receiver.yaml +++ b/examples/disable-persistence-queue-traces/rendered_manifests/configmap-cluster-receiver.yaml @@ -50,8 +50,12 @@ data: k8sattributes/metrics: extract: annotations: + - from: namespace + key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: pod key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: namespace key: splunk.com/metricsIndex tag_name: com.splunk.index diff --git a/examples/disable-persistence-queue-traces/rendered_manifests/daemonset.yaml b/examples/disable-persistence-queue-traces/rendered_manifests/daemonset.yaml index b3f2fbb34e..eb1632fae7 100644 --- a/examples/disable-persistence-queue-traces/rendered_manifests/daemonset.yaml +++ b/examples/disable-persistence-queue-traces/rendered_manifests/daemonset.yaml @@ -32,7 +32,7 @@ spec: component: otel-collector-agent release: default annotations: - checksum/config: 8f13ff66652ba33fe2a9aee4c1a2e43517bfe12d57093be5d675ef515f537d78 + checksum/config: 8cf566b711b895e8209d8c055fc25b2245e74abe84743cf64655a07cb78fee92 kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true diff --git a/examples/disable-persistence-queue-traces/rendered_manifests/deployment-cluster-receiver.yaml b/examples/disable-persistence-queue-traces/rendered_manifests/deployment-cluster-receiver.yaml index a39fcbcba6..97d49e26a4 100644 --- a/examples/disable-persistence-queue-traces/rendered_manifests/deployment-cluster-receiver.yaml +++ b/examples/disable-persistence-queue-traces/rendered_manifests/deployment-cluster-receiver.yaml @@ -31,7 +31,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: 31057c8fa63ede3a8e48bce75a40865619cdd244a7cc16ade3471013072fb603 + checksum/config: 130873286d49dc56b755805d556b3094fcb6696ce4e9326408389c58942d46db spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/examples/enable-persistence-queue/rendered_manifests/configmap-agent.yaml b/examples/enable-persistence-queue/rendered_manifests/configmap-agent.yaml index 00024b666c..887cc626b5 100644 --- a/examples/enable-persistence-queue/rendered_manifests/configmap-agent.yaml +++ b/examples/enable-persistence-queue/rendered_manifests/configmap-agent.yaml @@ -165,8 +165,12 @@ data: k8sattributes/metrics: extract: annotations: + - from: namespace + key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: pod key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: namespace key: splunk.com/metricsIndex tag_name: com.splunk.index diff --git a/examples/enable-persistence-queue/rendered_manifests/configmap-cluster-receiver.yaml b/examples/enable-persistence-queue/rendered_manifests/configmap-cluster-receiver.yaml index f8f09c8b36..7b89f46ffe 100644 --- a/examples/enable-persistence-queue/rendered_manifests/configmap-cluster-receiver.yaml +++ b/examples/enable-persistence-queue/rendered_manifests/configmap-cluster-receiver.yaml @@ -50,8 +50,12 @@ data: k8sattributes/metrics: extract: annotations: + - from: namespace + key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: pod key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: namespace key: splunk.com/metricsIndex tag_name: com.splunk.index diff --git a/examples/enable-persistence-queue/rendered_manifests/daemonset.yaml b/examples/enable-persistence-queue/rendered_manifests/daemonset.yaml index a90d194015..e2a6044fff 100644 --- a/examples/enable-persistence-queue/rendered_manifests/daemonset.yaml +++ b/examples/enable-persistence-queue/rendered_manifests/daemonset.yaml @@ -32,7 +32,7 @@ spec: component: otel-collector-agent release: default annotations: - checksum/config: 7e28ba091fe865d050d7328d4484a25a517e3f7937d660ab8aca68b1f7206d21 + checksum/config: 07fe1f32f371cae54b4ed702d65fd1a60a134f0c4d8b66dcfec1eb81ed1907bc kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true diff --git a/examples/enable-persistence-queue/rendered_manifests/deployment-cluster-receiver.yaml b/examples/enable-persistence-queue/rendered_manifests/deployment-cluster-receiver.yaml index a39fcbcba6..97d49e26a4 100644 --- a/examples/enable-persistence-queue/rendered_manifests/deployment-cluster-receiver.yaml +++ b/examples/enable-persistence-queue/rendered_manifests/deployment-cluster-receiver.yaml @@ -31,7 +31,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: 31057c8fa63ede3a8e48bce75a40865619cdd244a7cc16ade3471013072fb603 + checksum/config: 130873286d49dc56b755805d556b3094fcb6696ce4e9326408389c58942d46db spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/examples/multi-metrics/rendered_manifests/configmap-agent.yaml b/examples/multi-metrics/rendered_manifests/configmap-agent.yaml index 64fecd53cd..332bc8a97e 100644 --- a/examples/multi-metrics/rendered_manifests/configmap-agent.yaml +++ b/examples/multi-metrics/rendered_manifests/configmap-agent.yaml @@ -137,8 +137,12 @@ data: k8sattributes/metrics: extract: annotations: + - from: namespace + key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: pod key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: namespace key: splunk.com/metricsIndex tag_name: com.splunk.index diff --git a/examples/multi-metrics/rendered_manifests/configmap-cluster-receiver.yaml b/examples/multi-metrics/rendered_manifests/configmap-cluster-receiver.yaml index f8f09c8b36..7b89f46ffe 100644 --- a/examples/multi-metrics/rendered_manifests/configmap-cluster-receiver.yaml +++ b/examples/multi-metrics/rendered_manifests/configmap-cluster-receiver.yaml @@ -50,8 +50,12 @@ data: k8sattributes/metrics: extract: annotations: + - from: namespace + key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: pod key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: namespace key: splunk.com/metricsIndex tag_name: com.splunk.index diff --git a/examples/multi-metrics/rendered_manifests/daemonset.yaml b/examples/multi-metrics/rendered_manifests/daemonset.yaml index abe7bd8296..0c79cc9b63 100644 --- a/examples/multi-metrics/rendered_manifests/daemonset.yaml +++ b/examples/multi-metrics/rendered_manifests/daemonset.yaml @@ -32,7 +32,7 @@ spec: component: otel-collector-agent release: default annotations: - checksum/config: 5b4ddaf2cc99de360060d11ef3f956eaf152f249d2aeb62cf477602745b4cea1 + checksum/config: 03163bd786efbdb0c43d07cb64eb17cf6cd9595af1c5db2ccf2c4c84615a2dfb kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true diff --git a/examples/multi-metrics/rendered_manifests/deployment-cluster-receiver.yaml b/examples/multi-metrics/rendered_manifests/deployment-cluster-receiver.yaml index a39fcbcba6..97d49e26a4 100644 --- a/examples/multi-metrics/rendered_manifests/deployment-cluster-receiver.yaml +++ b/examples/multi-metrics/rendered_manifests/deployment-cluster-receiver.yaml @@ -31,7 +31,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: 31057c8fa63ede3a8e48bce75a40865619cdd244a7cc16ade3471013072fb603 + checksum/config: 130873286d49dc56b755805d556b3094fcb6696ce4e9326408389c58942d46db spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/examples/only-metrics-platform/rendered_manifests/configmap-agent.yaml b/examples/only-metrics-platform/rendered_manifests/configmap-agent.yaml index 4501f48092..d0c7df2ac4 100644 --- a/examples/only-metrics-platform/rendered_manifests/configmap-agent.yaml +++ b/examples/only-metrics-platform/rendered_manifests/configmap-agent.yaml @@ -110,8 +110,12 @@ data: k8sattributes/metrics: extract: annotations: + - from: namespace + key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: pod key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: namespace key: splunk.com/metricsIndex tag_name: com.splunk.index diff --git a/examples/only-metrics-platform/rendered_manifests/configmap-cluster-receiver.yaml b/examples/only-metrics-platform/rendered_manifests/configmap-cluster-receiver.yaml index f8f09c8b36..7b89f46ffe 100644 --- a/examples/only-metrics-platform/rendered_manifests/configmap-cluster-receiver.yaml +++ b/examples/only-metrics-platform/rendered_manifests/configmap-cluster-receiver.yaml @@ -50,8 +50,12 @@ data: k8sattributes/metrics: extract: annotations: + - from: namespace + key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: pod key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype - from: namespace key: splunk.com/metricsIndex tag_name: com.splunk.index diff --git a/examples/only-metrics-platform/rendered_manifests/daemonset.yaml b/examples/only-metrics-platform/rendered_manifests/daemonset.yaml index 386aeb8542..fc8119acfe 100644 --- a/examples/only-metrics-platform/rendered_manifests/daemonset.yaml +++ b/examples/only-metrics-platform/rendered_manifests/daemonset.yaml @@ -32,7 +32,7 @@ spec: component: otel-collector-agent release: default annotations: - checksum/config: 1d48c12fc0681c6793c2c6e581e5199248d33a81165761431e28cc813cecd5c5 + checksum/config: c3dbc1971b6dfeee15734ef1758d5b78e03f60fa6ce35f5d58b4df90ea71c24d kubectl.kubernetes.io/default-container: otel-collector spec: hostNetwork: true diff --git a/examples/only-metrics-platform/rendered_manifests/deployment-cluster-receiver.yaml b/examples/only-metrics-platform/rendered_manifests/deployment-cluster-receiver.yaml index a39fcbcba6..97d49e26a4 100644 --- a/examples/only-metrics-platform/rendered_manifests/deployment-cluster-receiver.yaml +++ b/examples/only-metrics-platform/rendered_manifests/deployment-cluster-receiver.yaml @@ -31,7 +31,7 @@ spec: component: otel-k8s-cluster-receiver release: default annotations: - checksum/config: 31057c8fa63ede3a8e48bce75a40865619cdd244a7cc16ade3471013072fb603 + checksum/config: 130873286d49dc56b755805d556b3094fcb6696ce4e9326408389c58942d46db spec: serviceAccountName: default-splunk-otel-collector nodeSelector: diff --git a/helm-charts/splunk-otel-collector/templates/config/_common.tpl b/helm-charts/splunk-otel-collector/templates/config/_common.tpl index 9e40a9206a..1409739c36 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_common.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_common.tpl @@ -232,6 +232,10 @@ k8sattributes/clusterReceiver: {{- if eq (include "splunk-otel-collector.splunkPlatformEnabled" .) "true"}} annotations: - key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype + from: namespace + - key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype from: pod - key: splunk.com/index tag_name: com.splunk.index @@ -266,6 +270,10 @@ k8sattributes/metrics: metadata: [] annotations: - key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype + from: namespace + - key: splunk.com/sourcetype + tag_name: com.splunk.sourcetype from: pod - key: splunk.com/metricsIndex tag_name: com.splunk.index @@ -338,6 +346,20 @@ resource/logs: {{- end }} {{- end }} +{{/* +Resource processor for metrics manipulations +*/}} +{{- define "splunk-otel-collector.resourceMetricsProcessor" -}} +resource/metrics: + attributes: + {{- if .Values.splunkPlatform.sourcetype }} + # Insert the sourcetype value from values.yaml if it has not already been set through annotations. + - key: com.splunk.sourcetype + value: "{{.Values.splunkPlatform.sourcetype }}" + action: insert + {{- end }} +{{- end }} + {{/* The transform processor adds service.name attribute to logs the same way as it's done by istio for the generated traces https://github.com/istio/istio/blob/6237cb4e63cf9a332327cc0a815d6b46257e6f8a/pkg/config/analysis/analyzers/testdata/common/sidecar-injector-configmap.yaml#L110-L115 @@ -435,9 +457,6 @@ splunk_hec/platform_metrics: token: "${SPLUNK_PLATFORM_HEC_TOKEN}" index: {{ .Values.splunkPlatform.metricsIndex | quote }} source: {{ .Values.splunkPlatform.source | quote }} - {{- if .Values.splunkPlatform.sourcetype }} - sourcetype: {{ .Values.splunkPlatform.sourcetype | quote }} - {{- end }} max_idle_conns: {{ .Values.splunkPlatform.maxConnections }} max_idle_conns_per_host: {{ .Values.splunkPlatform.maxConnections }} disable_compression: {{ .Values.splunkPlatform.disableCompression }} diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl index 501cb27717..b069d45a05 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-agent.tpl @@ -648,6 +648,9 @@ processors: {{- include "splunk-otel-collector.k8sAttributesSplunkPlatformMetrics" . | nindent 2 }} filter: node_from_env_var: K8S_NODE_NAME + {{- if .Values.splunkPlatform.sourcetype }} + {{- include "splunk-otel-collector.resourceMetricsProcessor" . | nindent 2 }} + {{- end }} {{- end }} {{- if eq .Values.logsEngine "fluentd" }} @@ -1026,6 +1029,9 @@ service: {{- end }} {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }} - k8sattributes/metrics + {{- if .Values.splunkPlatform.sourcetype }} + - resource/metrics + {{- end }} {{- end }} exporters: {{- if $gatewayEnabled }} @@ -1052,6 +1058,9 @@ service: - resource {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }} - k8sattributes/metrics + {{- if .Values.splunkPlatform.sourcetype }} + - resource/metrics + {{- end }} {{- end }} exporters: {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }} diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl index 16a31b00c6..682df4a33f 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-collector.tpl @@ -33,6 +33,9 @@ processors: {{- include "splunk-otel-collector.k8sAttributesSplunkPlatformMetrics" . | nindent 2 }} filter: node_from_env_var: K8S_NODE_NAME + {{- if .Values.splunkPlatform.sourcetype }} + {{- include "splunk-otel-collector.resourceMetricsProcessor" . | nindent 2 }} + {{- end }} {{- end }} {{- include "splunk-otel-collector.resourceLogsProcessor" . | nindent 2 }} @@ -207,6 +210,9 @@ service: {{- end }} {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }} - k8sattributes/metrics + {{- if .Values.splunkPlatform.sourcetype }} + - resource/metrics + {{- end }} {{- end }} exporters: {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }} @@ -261,6 +267,9 @@ service: {{- end }} {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }} - k8sattributes/metrics + {{- if .Values.splunkPlatform.sourcetype }} + - resource/metrics + {{- end }} {{- end }} exporters: {{- if (eq (include "splunk-otel-collector.splunkO11yEnabled" .) "true") }} diff --git a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl index 56df2992e5..3286ba7dab 100644 --- a/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl +++ b/helm-charts/splunk-otel-collector/templates/config/_otel-k8s-cluster-receiver-config.tpl @@ -79,6 +79,9 @@ processors: {{- include "splunk-otel-collector.k8sAttributesSplunkPlatformMetrics" . | nindent 2 }} filter: node_from_env_var: K8S_NODE_NAME + {{- if .Values.splunkPlatform.sourcetype }} + {{- include "splunk-otel-collector.resourceMetricsProcessor" . | nindent 2 }} + {{- end }} {{- end }} batch: @@ -237,6 +240,9 @@ service: - resource {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }} - k8sattributes/metrics + {{- if .Values.splunkPlatform.sourcetype }} + - resource/metrics + {{- end }} {{- end }} - resource/k8s_cluster exporters: @@ -259,6 +265,9 @@ service: - resource {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }} - k8sattributes/metrics + {{- if .Values.splunkPlatform.sourcetype }} + - resource/metrics + {{- end }} {{- end }} exporters: {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }} @@ -280,6 +289,9 @@ service: - resource {{- if (eq (include "splunk-otel-collector.platformMetricsEnabled" $) "true") }} - k8sattributes/metrics + {{- if .Values.splunkPlatform.sourcetype }} + - resource/metrics + {{- end }} {{- end }} exporters: {{- if (eq (include "splunk-otel-collector.o11yMetricsEnabled" .) "true") }} diff --git a/test/k8s_logging_tests/test_config_logging.py b/test/k8s_logging_tests/test_config_logging.py index ed83846c4c..9b1ee48e8d 100644 --- a/test/k8s_logging_tests/test_config_logging.py +++ b/test/k8s_logging_tests/test_config_logging.py @@ -81,3 +81,24 @@ def test_metric_index_from_annotations(setup, index, expected): logger.info("Splunk received %s events in the last minute", len(events)) assert len(events) >= expected + +@pytest.mark.parametrize("index,sourcetype,expected", [ + ("test_metrics", "sourcetype-anno", 1) +]) +def test_metric_sourcetype_from_annotations(setup, index, sourcetype, expected): + + ''' + Test that metrics are being assigned the "sourcetype-anno" sourcetype, as defined by splunk.com/sourcetype annotation added during setup + ''' + logger.info("testing for metrics index={0} sourcetype={1} expected={2} event(s)".format(index, sourcetype, expected)) + search_query = "index={0} filter=\"sourcetype={1}\"".format(index, sourcetype) + + events = check_events_from_splunk(start_time="-1h@h", + url=setup["splunkd_url"], + user=setup["splunk_user"], + query=["mpreview {0}".format( + search_query)], + password=setup["splunk_password"]) + logger.info("Splunk received %s events in the last minute", + len(events)) + assert len(events) >= expected \ No newline at end of file diff --git a/test/test_setup.yaml b/test/test_setup.yaml index 82434fa2d4..ce45e8446d 100644 --- a/test/test_setup.yaml +++ b/test/test_setup.yaml @@ -26,6 +26,7 @@ metadata: name: default annotations: splunk.com/metricsIndex: test_metrics + splunk.com/sourcetype: sourcetype-anno --- apiVersion: batch/v1 kind: Job