Merge pull request #562 from logzio/feature/set-servicegraph-flush-in…

…terval

[Feature] set servicegraph flush interval

.github/workflows/logzio-monitoring-test.yaml

@@ -148,6 +148,12 @@ jobs:
           kubectl apply -f tests/resources/tracegen-monitoring.yaml
           kubectl rollout status deployment/trace-gen --timeout=300s
 
+      - name: Run otel demo
+        run: |
+          helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+          helm repo update
+          helm install otel-demo -f tests/resources/otel-demo-monitoring.yaml open-telemetry/opentelemetry-demo --version 0.32.5
+          kubectl rollout status deployment/otel-demo-loadgenerator --timeout=300s
       - name: Sleep
         run: sleep 180
 

.github/workflows/logzio-telemetry-test.yaml

@@ -56,6 +56,7 @@ jobs:
           helm upgrade --install \
           --set traces.enabled=true \
           --set spm.enabled=true \
+          --set serviceGraph.enabled=true \
           --set metrics.enabled=true \
           --set secrets.TracesToken=${{ secrets.LOGZIO_TRACES_TOKEN }} \
           --set secrets.SpmToken=${{ secrets.LOGZIO_METRICS_TOKEN }} \
@@ -72,7 +73,13 @@ jobs:
         run: |
           kubectl apply -f tests/resources/tracegen.yaml
           kubectl rollout status deployment/trace-gen --timeout=300s
-
+      - name: Run otel demo
+        run: |
+          helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
+          helm repo update
+          helm install otel-demo -f tests/resources/otel-demo.yaml open-telemetry/opentelemetry-demo --version 0.32.5
+          kubectl rollout status deployment/otel-demo-loadgenerator --timeout=300s
+  
       - name: sleep for 3 minutes
         run: sleep 180
 

charts/logzio-telemetry/Chart.yaml

@@ -25,7 +25,7 @@ dependencies:
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
 
-version: 4.2.9
+version: 4.3.0
 
 
 # This is the version number of the application being deployed. This version number should be

charts/logzio-telemetry/README.md

@@ -412,6 +412,8 @@ If you don't want the sub charts to installed add the relevant flag per sub char
 
 
 ## Change log
+* 4.3.0
+  - Set `servicegraph` connector, `metrics_flush_interval` setting to `60s` to reduce outgoing connections
 * 4.2.9
   - Add batch processor to the SPM pipeline, to reduce stress and increase efficiency.
 * 4.2.8
@@ -467,6 +469,12 @@ If you don't want the sub charts to installed add the relevant flag per sub char
       - `kubeStateMetrics.enabled`
       - `pushGateway.enabled`
       - `nodeExporter.enabled`
+
+
+
+<details>
+  <summary markdown="span"> Expand to check old versions </summary>
+
 * 2.2.0
   - Upgraded SPM collector image to version `0.80.0`.
   - Added service graph connector metrics.
@@ -479,11 +487,6 @@ If you don't want the sub charts to installed add the relevant flag per sub char
     - Add `unified_status_code` dimension
       - Takes value of `rpc_grpc_status_code` / `http_status_code`
   - Add `containerSecurityContext` configuration option for container based policies. 
-
-
-<details>
-  <summary markdown="span"> Expand to check old versions </summary>
-
 * 2.0.0
   - Upgrade sub charts to their latest versions.
     - `kube-state-metrics` to `4.24.0`

charts/logzio-telemetry/values.yaml

@@ -700,6 +700,7 @@ serviceGraph:
         store:
           ttl: 5s
           max_items: 100000
+        metrics_flush_interval: 60s
     service:
       pipelines:
         traces:

tests/metrics_e2e_test.go

@@ -3,13 +3,14 @@ package tests
 import (
 	"encoding/json"
 	"fmt"
-	"go.uber.org/zap"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
 	"strings"
 	"testing"
+
+	"go.uber.org/zap"
 )
 
 // MetricResponse represents the structure of the API response
@@ -38,6 +39,22 @@ func TestContainerMetrics(t *testing.T) {
 	testMetrics(t, requiredMetrics, escapedQuery)
 }
 
+func TestServiceGraphMetrics(t *testing.T) {
+	requiredMetrics := map[string][]string{
+		"traces_service_graph_request_total":                 {"client", "server"},
+		"traces_service_graph_request_failed_total":          {"client", "server"},
+		"traces_service_graph_request_server_seconds_bucket": {"client", "server"},
+		"traces_service_graph_request_server_seconds_count":  {"client", "server"},
+		"traces_service_graph_request_server_seconds_sum":    {"client", "server"},
+		"traces_service_graph_request_client_seconds_bucket": {"client", "server"},
+		"traces_service_graph_request_client_seconds_count":  {"client", "server"},
+		"traces_service_graph_request_client_seconds_sum":    {"client", "server"},
+	}
+	envId := os.Getenv("ENV_ID")
+	query := fmt.Sprintf(`{client_env_id='%s'}`, envId)
+	testMetrics(t, requiredMetrics, query)
+}
+
 func TestInfrastructureMetrics(t *testing.T) {
 	if os.Getenv("KUBERNETES_ENV") == "eks-fargate" {
 		t.Skip("Skipping infrastructure metrics test")

tests/resources/otel-demo-monitoring.yaml

@@ -0,0 +1,18 @@
+
+default:
+  envOverrides:
+    - name: OTEL_COLLECTOR_NAME
+      value: logzio-monitoring-otel-collector.monitoring.svc.cluster.local
+
+opentelemetry-collector:
+  enabled: false
+
+jaeger:
+  enabled: false
+
+prometheus:
+  enabled: false
+
+grafana:
+  enabled: false
+

tests/resources/otel-demo.yaml

@@ -0,0 +1,18 @@
+
+default:
+  envOverrides:
+    - name: OTEL_COLLECTOR_NAME
+      value: logzio-k8s-telemetry-otel-collector.default.svc.cluster.local
+
+opentelemetry-collector:
+  enabled: false
+
+jaeger:
+  enabled: false
+
+prometheus:
+  enabled: false
+
+grafana:
+  enabled: false
+