diff --git a/.github/workflows/helm-prometheus.yaml b/.github/workflows/helm-prometheus.yaml new file mode 100644 index 00000000..bc11e06c --- /dev/null +++ b/.github/workflows/helm-prometheus.yaml @@ -0,0 +1,43 @@ +name: Helm GitHub Actions for Prometheus + +on: + pull_request: + paths: + - base-helm-configs/prometheus/** + - base-kustomize/prometheus/** + - .github/workflows/helm-prometheus.yaml +jobs: + helm: + strategy: + matrix: + overlays: + - base + name: Helm + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - uses: azure/setup-helm@v3 + with: + version: latest + token: "${{ secrets.GITHUB_TOKEN }}" + id: helm + - name: Add prometheus-community repo to helm + run: | + ${{ steps.helm.outputs.helm-path }} repo add prometheus-community https://prometheus-community.github.io/helm-charts + ${{ steps.helm.outputs.helm-path }} repo update + - name: Run Helm Template + run: | + ${{ steps.helm.outputs.helm-path }} template prometheus prometheus-community/kube-prometheus-stack \ + --create-namespace \ + --namespace=prometheus \ + -f ${{ github.workspace }}//base-helm-configs/prometheus/prometheus-helm-overrides.yaml \ + -f ${{ github.workspace }}//base-helm-configs/prometheus/alerting_rules.yaml \ + -f ${{ github.workspace }}//base-helm-configs/prometheus/alertmanager_config.yaml \ + --post-renderer ${{ github.workspace }}/base-kustomize/kustomize.sh \ + --post-renderer-args prometheus/${{ matrix.overlays }} > /tmp/rendered.yaml + - name: Return helm Build + uses: actions/upload-artifact@v4 + with: + name: helm-prometheus-artifact-${{ matrix.overlays }} + path: /tmp/rendered.yaml diff --git a/.github/workflows/kustomize-prometheus.yaml b/.github/workflows/kustomize-prometheus.yaml deleted file mode 100644 index 59fccfff..00000000 --- a/.github/workflows/kustomize-prometheus.yaml +++ /dev/null @@ -1,33 +0,0 @@ -name: Kustomize GitHub Actions for Prometheus - -on: - pull_request: - paths: - - base-kustomize/prometheus/** - - .github/workflows/kustomize-prometheus.yaml -jobs: - kustomize: - name: Kustomize - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - uses: azure/setup-helm@v3 - with: - version: latest - token: "${{ secrets.GITHUB_TOKEN }}" - id: helm - - name: Kustomize Install - working-directory: /usr/local/bin/ - run: | - if [ ! -f /usr/local/bin/kustomize ]; then - curl -s "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" | sudo bash - fi - - name: Run Kustomize Build - run: | - kustomize build base-kustomize/prometheus/ --enable-helm --helm-command ${{ steps.helm.outputs.helm-path }} > /tmp/rendered.yaml - - name: Return Kustomize Build - uses: actions/upload-artifact@v4 - with: - name: kustomize-prometheus-artifact - path: /tmp/rendered.yaml diff --git a/base-kustomize/prometheus/alerting_rules.yaml b/base-helm-configs/prometheus/alerting_rules.yaml similarity index 100% rename from base-kustomize/prometheus/alerting_rules.yaml rename to base-helm-configs/prometheus/alerting_rules.yaml diff --git a/base-kustomize/prometheus/alertmanager_config.yaml b/base-helm-configs/prometheus/alertmanager_config.yaml similarity index 100% rename from base-kustomize/prometheus/alertmanager_config.yaml rename to base-helm-configs/prometheus/alertmanager_config.yaml diff --git a/base-kustomize/prometheus/values.yaml b/base-helm-configs/prometheus/prometheus-helm-overrides.yaml similarity index 100% rename from base-kustomize/prometheus/values.yaml rename to base-helm-configs/prometheus/prometheus-helm-overrides.yaml diff --git a/base-kustomize/prometheus/base/kustomization.yaml b/base-kustomize/prometheus/base/kustomization.yaml new file mode 100644 index 00000000..34f7f9bd --- /dev/null +++ b/base-kustomize/prometheus/base/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - all.yaml diff --git a/base-kustomize/prometheus/kustomization.yaml b/base-kustomize/prometheus/kustomization.yaml deleted file mode 100644 index ead3b609..00000000 --- a/base-kustomize/prometheus/kustomization.yaml +++ /dev/null @@ -1,13 +0,0 @@ -resources: - - ns-prometheus.yaml - -helmCharts: - - name: kube-prometheus-stack - repo: https://prometheus-community.github.io/helm-charts - releaseName: kube-prometheus-stack - namespace: prometheus - includeCRDs: true - valuesFile: values.yaml - additionalValuesFiles: - - alerting_rules.yaml - - alertmanager_config.yaml diff --git a/base-kustomize/prometheus/ns-prometheus.yaml b/base-kustomize/prometheus/ns-prometheus.yaml deleted file mode 100644 index 968d863f..00000000 --- a/base-kustomize/prometheus/ns-prometheus.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - labels: - kubernetes.io/metadata.name: prometheus - name: prometheus - name: prometheus diff --git a/bin/install-prometheus.sh b/bin/install-prometheus.sh new file mode 100644 index 00000000..4cf33df4 --- /dev/null +++ b/bin/install-prometheus.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +helm repo add prometheus-community https://prometheus-community.github.io/helm-charts +helm repo update +helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \ + --create-namespace --namespace=prometheus --timeout 10m \ + -f /opt/genestack/base-helm-configs/prometheus/prometheus-helm-overrides.yaml \ + -f /etc/genestack/helm-configs/prometheus/prometheus-helm-overrides.yaml \ + -f /opt/genestack/base-helm-configs/prometheus/alerting_rules.yaml \ + -f /etc/genestack/helm-configs/prometheus/alerting_rules.yaml \ + -f /opt/genestack/base-helm-configs/prometheus/alertmanager_config.yaml \ + -f /etc/genestack/helm-configs/prometheus/alertmanager_config.yaml \ + --post-renderer /opt/genestack/base-kustomize/kustomize.sh \ + --post-renderer-args prometheus/base "$@" diff --git a/docs/prometheus.md b/docs/prometheus.md index 583bfe4b..d5946695 100644 --- a/docs/prometheus.md +++ b/docs/prometheus.md @@ -1,27 +1,47 @@ # Prometheus -We are taking advantage of the prometheus community kube-prometheus-stack as well as other various components for monitoring and alerting. For more information take a look at [Prometheus Kube Stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack) +We are taking advantage of the prometheus community kube-prometheus-stack as +well as other various components for monitoring and alerting. For more +information, take a look at [Prometheus Kube Stack](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack). -#### Install kube-prometheus-stack helm chart +!!! tip -## Update Alertmanager configurations + You may need to provide custom values to configure prometheus. For a simple + single region or lab deployment you can supply an additional overrides flag + using the example found at `base-helm-configs/aio-example-openstack-overrides.yaml`. + In other cases such as a multi-region deployment you may want to view the + [Multi-Region Support](multi-region-support.md) guide to for a workflow + solution. -Currently you can supply a Teams webhook url to send all current alerts to a teams channel. This section will be updated to be more comprehensive in the future... +## Install the Prometheus Stack -!!! tip +!!! example "Run the Prometheus deployment Script `bin/install-prometheus.sh`" - You can ignore this step if you don't want to send alerts to teams, the alertmanager will still deploy and provide information + ``` shell + --8<-- "bin/install-prometheus.sh" + ``` -``` shell -webhook_url='https://my.webhook.example' -sed -i -e "s#https://webhook_url.example#$webhook_url#" /etc/genestack/kustomize/prometheus/alertmanager_config.yaml -``` +!!! success -## Install the prometheus stack + If the installation is successful, you should see the related exporter pods + in the prometheus namespace. + ``` shell + kubectl -n prometheus get pods -l "release=prometheus" + ``` -``` shell -kubectl kustomize --enable-helm /etc/genestack/kustomize/prometheus | kubectl apply --server-side -f - -``` +## Update Alertmanager Configuration -!!! success - If the installation is successful, you should see the related exporter pods in the prometheus namespace. +In this example, we supply a Teams webhook URL to send all open alerts to a +teams channel. However, there are a plethora of other receivers available. +For a full list, review prometheus documentation: [receiver-integration-settings](https://prometheus.io/docs/alerting/latest/configuration/#receiver-integration-settings). + +!!! example + + You can ignore this step if you don't want to send alerts to Teams, the + alertmanager will still deploy and provide information. + + ``` shell + read -p "webhook_url: " webhook_url; + sed -i -e "s#https://webhook_url.example#$webhook_url#" \ + /etc/genestack/helm-configs/prometheus/alertmanager_config.yaml + ```