Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(mixin): Add pod prefix config option (non-SSD) #14624

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

feat(mixin): Add pod prefix config option (non-SSD) #14624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
a908ef9
feat(mixin): Add pod prefix config option (non-SSD)
mtweten Oct 25, 2024
212d217
Run make loki-mixin
mtweten Oct 25, 2024
5fc49a9
Fixup rebase
mtweten Nov 27, 2024
e36a46a
Merge branch 'main' into main
mtweten Dec 2, 2024
5162d3f
Merge branch 'main' into main
mtweten Dec 2, 2024
96c937c
Merge branch 'main' into main
mtweten Dec 3, 2024
82d57d6
Add pod prefix to partition-ingester
mtweten Jan 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@
"span": 4,
"targets": [
{
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
"format": "time_series",
"legendFormat": "{{pod}}",
"legendLink": null
Expand Down Expand Up @@ -426,7 +426,7 @@
"span": 4,
"targets": [
{
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
"format": "time_series",
"legendFormat": " {{pod}} ",
"legendLink": null
Expand Down Expand Up @@ -579,7 +579,7 @@
"span": 6,
"targets": [
{
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
"format": "time_series",
"legendFormat": "{{user}}",
"legendLink": null
Expand All @@ -606,7 +606,7 @@
"span": 6,
"targets": [
{
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
"refId": "A"
}
],
Expand All @@ -619,7 +619,7 @@
"span": 6,
"targets": [
{
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
"refId": "A"
}
],
Expand Down
10 changes: 5 additions & 5 deletions production/loki-mixin-compiled/dashboards/loki-deletion.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -379,7 +379,7 @@
"span": 4,
"targets": [
{
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
"format": "time_series",
"legendFormat": "{{pod}}",
"legendLink": null
Expand Down Expand Up @@ -426,7 +426,7 @@
"span": 4,
"targets": [
{
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
"format": "time_series",
"legendFormat": " {{pod}} ",
"legendLink": null
Expand Down Expand Up @@ -579,7 +579,7 @@
"span": 6,
"targets": [
{
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
"format": "time_series",
"legendFormat": "{{user}}",
"legendLink": null
Expand All @@ -606,7 +606,7 @@
"span": 6,
"targets": [
{
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
"refId": "A"
}
],
Expand All @@ -619,7 +619,7 @@
"span": 6,
"targets": [
{
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor.*|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
"refId": "A"
}
],
Expand Down
4 changes: 4 additions & 0 deletions production/loki-mixin/config.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,10 @@
per_namespace_label: 'namespace',
per_job_label: 'job',

// Optional prefix for pod matchers (non SSD). Useful for Loki distributed installs via helm which prefix pod names with the helm release name.
// When specifying the prefix, make sure to include the trailing hyphen, e.g. "loki-".
pod_prefix_matcher: '',

// Grouping labels, to uniquely identify and group by {jobs, clusters}
job_labels: [$._config.per_cluster_label, $._config.per_namespace_label, $._config.per_job_label],
cluster_labels: [$._config.per_cluster_label, $._config.per_namespace_label],
Expand Down
2 changes: 1 addition & 1 deletion production/loki-mixin/dashboards/loki-deletion.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ local g = import 'grafana-builder/grafana.libsonnet';
local utils = import 'mixin-utils/utils.libsonnet';

(import 'dashboard-utils.libsonnet') {
local compactor_matcher = 'pod=~"(compactor|%s-backend.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher,
local compactor_matcher = 'pod=~"(%scompactor.*|%s-backend.*|loki-single-binary)"' % [$._config.pod_prefix_matcher, $._config.ssd.pod_prefix_matcher],
Copy link
Author

@mtweten mtweten Oct 25, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I also added a missing .* in this selector to the end of compactor, which is why the generated dashboards above have changed. I believe this was a defect - see the equivalent matcher in loki-retention.libsonnet which already has the .*.

loki/production/loki-mixin/dashboards/loki-retention.libsonnet

Line 3 in 51c42e8

then 'pod=~"(compactor.*|%s-backend.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher

grafanaDashboards+::
{
'loki-deletion.json':
Expand Down
12 changes: 6 additions & 6 deletions production/loki-mixin/dashboards/loki-operational.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,14 @@ local utils = import 'mixin-utils/utils.libsonnet';
podMatchers:: {
cortexgateway: [utils.selector.re('pod', 'cortex-gw')],
distributor: if $._config.meta_monitoring.enabled
then [utils.selector.re('pod', '(distributor|%s-write|loki-single-binary)' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-write.*' % $._config.ssd.pod_prefix_matcher else 'distributor.*'))],
then [utils.selector.re('pod', '(%sdistributor|%s-write|loki-single-binary)' % [$._config.pod_prefix_matcher, $._config.ssd.pod_prefix_matcher])]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-write.*' % $._config.ssd.pod_prefix_matcher else '%sdistributor.*' % $._config.pod_prefix_matcher))],
ingester: if $._config.meta_monitoring.enabled
then [utils.selector.re('pod', '(partition-ingester.*|ingester.*|%s-write|loki-single-binary)' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-write.*' % $._config.ssd.pod_prefix_matcher else '(ingester.*|partition-ingester.*)'))],
then [utils.selector.re('pod', '(%(pod_prefix)spartition-ingester.*|%(pod_prefix)singester|%(ssd_prefix)s-write|loki-single-binary)' % {pod_prefix: $._config.pod_prefix_matcher, ssd_prefix: $._config.ssd.pod_prefix_matcher})]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-write.*' % $._config.ssd.pod_prefix_matcher else '(%(pod_prefix)singester.*|%(pod_prefix)spartition-ingester.*)' % {pod_prefix: $._config.pod_prefix_matcher} ))],
querier: if $._config.meta_monitoring.enabled
then [utils.selector.re('pod', '(querier|%s-read|loki-single-binary)' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-read.*' % $._config.ssd.pod_prefix_matcher else 'querier.*'))],
then [utils.selector.re('pod', '(%squerier|%s-read|loki-single-binary)' % [$._config.pod_prefix_matcher, $._config.ssd.pod_prefix_matcher])]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-read.*' % $._config.ssd.pod_prefix_matcher else '%squerier.*' % $._config.pod_prefix_matcher))],
backend: [utils.selector.re('pod', '%s-backend.*' % $._config.ssd.pod_prefix_matcher)],
},
}
Expand Down
4 changes: 2 additions & 2 deletions production/loki-mixin/dashboards/loki-reads-resources.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
(import 'dashboard-utils.libsonnet') {
local index_gateway_pod_matcher = if $._config.meta_monitoring.enabled
then 'container=~"loki|index-gateway", pod=~"(index-gateway.*|loki-single-binary)"'
then 'container=~"loki|index-gateway", pod=~"(%sindex-gateway.*|loki-single-binary)"' % $._config.pod_prefix_matcher
else 'container="index-gateway"',
local index_gateway_job_matcher = if $._config.meta_monitoring.enabled
then '(index-gateway.*|loki-single-binary)'
else 'index-gateway',

local ingester_pod_matcher = if $._config.meta_monitoring.enabled
then 'container=~"loki|ingester|partition-ingester", pod=~"(ingester.*|partition-ingester.*|loki-single-binary)"'
then 'container=~"loki|ingester|partition-ingester", pod=~"(%(pod_prefix)singester.*|%(pod_prefix)spartition-ingester.*|loki-single-binary)"' % {pod_prefix: $._config.pod_prefix_matcher}
else 'container=~"ingester|partition-ingester"',
local ingester_job_matcher = if $._config.meta_monitoring.enabled
then '(ingester.*|partition-ingester.*|loki-single-binary)'
Expand Down
2 changes: 1 addition & 1 deletion production/loki-mixin/dashboards/loki-retention.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
(import 'dashboard-utils.libsonnet') {
local compactor_pod_matcher = if $._config.meta_monitoring.enabled
then 'pod=~"(compactor.*|%s-backend.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher
then 'pod=~"(%scompactor.*|%s-backend.*|loki-single-binary)"' % [$._config.pod_prefix_matcher, $._config.ssd.pod_prefix_matcher]
else if $._config.ssd.enabled then 'container="loki", pod=~"%s-read.*"' % $._config.ssd.pod_prefix_matcher else 'container="compactor"',
local compactor_job_matcher = if $._config.meta_monitoring.enabled
then '"(compactor|%s-backend.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher
Expand Down
2 changes: 1 addition & 1 deletion production/loki-mixin/dashboards/loki-writes-resources.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
(import 'dashboard-utils.libsonnet') {
local ingester_pod_matcher = if $._config.meta_monitoring.enabled
then 'container=~"loki|ingester|partition-ingester", pod=~"(ingester.*|partition-ingester.*|loki-single-binary)"'
then 'container=~"loki|ingester|partition-ingester", pod=~"(%(pod_prefix)singester.*|%(pod_prefix)spartition-ingester.*|loki-single-binary)"' % {pod_prefix: $._config.pod_prefix_matcher}
else 'container=~"ingester|partition-ingester"',
local ingester_job_matcher = if $._config.meta_monitoring.enabled
then '(ingester.*|partition-ingester.*|loki-single-binary)'
Expand Down
Loading