Merge pull request #120 from infrawatch/import/stf153

Import master to stable-1.5
infrawatch · Oct 26, 2023 · 84cf82e · 84cf82e
2 parents c823d1f + f1c2bf4
commit 84cf82e
Show file tree

Hide file tree

Showing 38 changed files with 1,023 additions and 186 deletions.
diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -16,17 +16,16 @@ env:
 on: [push, pull_request]
 
 jobs:
-  metrics:
-    name: "[metrics] handler: ceilometer-metrics, collectd-metrics; application: prometheus"
-    runs-on: ubuntu-20.04
+  collectd-metrics-bridge:
+    name: "[metrics] transport: socket(sg-bridge); handler: collectd-metrics; application: prometheus"
+    runs-on: ubuntu-22.04
     env:
-      QDR_CHANNEL_CEILOMTR: ceilometer/metering.sample
-      QDR_CHANNEL_COLLECTD: collectd/metrics
+      QDR_CHANNEL: collectd/metrics
       BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
       PROMETHEUS_IMAGE: prom/prometheus:latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Prepare environment
         run: |
           mkdir -p /opt/stack/
@@ -39,6 +38,13 @@ jobs:
           sudo rm -rf /etc/mysql
           sudo dpkg -l | grep -i mysql
           sudo apt-get clean
+      - name: Prepare environment for postgres-server installation
+        run: |
+          sudo apt remove postgresql-client-common
+          sudo apt install postgresql-client-common=238
+          sudo apt install postgresql-common
+          sudo python -m pip install --upgrade pip
+          sudo python -m pip install --upgrade virtualenv
       - name: Install devstack
         run: |
           SOURCE=$(pwd)
@@ -58,17 +64,111 @@ jobs:
         run: |
           echo "${GITHUB_REF#refs/heads/}"
           git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})"
-      - name: Start sg-bridge for collectd from container image
+      - name: Start sg-bridge from container image
         if: steps.bridge_branch.outcome != 'success'
         run: |
           docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \
-            $BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL_COLLECTD \
+            $BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \
             --gw_unix=$BRIDGE_SOCKET
-      - name: Start sg-bridge for collectd with same branch
+      - name: Start sg-bridge from same branch
         if: steps.bridge_branch.outcome == 'success'
         run: |
           docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
-            -e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL_COLLECTD -e OPSTOOLS_REPO \
+            -e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
+            --workdir=$(dirname $BRIDGE_SOCKET) \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh
+      - name: Install collectd
+        run: |
+          sudo apt-get install collectd
+          sudo systemctl stop collectd && sudo systemctl disable collectd
+          sudo cp ci/integration/metrics/collectd/collectd.conf /etc/collectd/collectd.conf
+          sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log
+          sudo collectd -C ci/integration/metrics/collectd/collectd.conf
+      - name: Run sg-core to process metrics
+        run: |
+          docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
+            --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_sg.sh
+      - name: Run Prometheus to store metrics
+        run: |
+          docker run --name=prometheus -d --network host \
+          --volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \
+          $PROMETHEUS_IMAGE
+      - name: Debug output
+        run: |
+          sleep 360
+          echo "=========================== qdr =========================" && \
+          docker exec qdr qdstat -b 127.0.0.1:5666 -a
+          docker logs qdr
+          echo "========================= sg-core =======================" && \
+          docker logs sgcore
+          echo "======================== prometheus =====================" && \
+          docker logs prometheus
+      - name: Validate metrics processing
+        run: |
+          docker run --name=validate -uroot --network host \
+            --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/collectd/run_validation.sh
+#-------------------------------------------------------------------------------
+  ceilometer-metrics-bridge:
+    name: "[metrics] transport: socket(sg-bridge); handler: ceilometer-metrics; application: prometheus"
+    runs-on: ubuntu-22.04
+    env:
+      QDR_CHANNEL: anycast/ceilometer/metering.sample
+      BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
+      PROMETHEUS_IMAGE: prom/prometheus:latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Prepare environment
+        run: |
+          mkdir -p /opt/stack/
+          sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack
+      - name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366
+        run: |
+          sudo apt-get -f install -o Dpkg::Options::="--force-overwrite"
+          sudo apt-get purge mysql\*
+          sudo rm -rf /var/lib/mysql
+          sudo rm -rf /etc/mysql
+          sudo dpkg -l | grep -i mysql
+          sudo apt-get clean
+      - name: Prepare environment for postgres-server installation
+        run: |
+          sudo apt remove postgresql-client-common
+          sudo apt install postgresql-client-common=238
+          sudo apt install postgresql-common
+          sudo python -m pip install --upgrade pip
+          sudo python -m pip install --upgrade virtualenv
+      - name: Install devstack
+        run: |
+          SOURCE=$(pwd)
+          git clone http://github.com/openstack/devstack /opt/stack/devstack
+          pushd /opt/stack/devstack
+          cp $SOURCE/ci/integration/metrics/local.conf .
+          sudo apt-get update
+          ./stack.sh
+          popd
+      # start message bus services
+      - name: Start QDR service
+        run: |
+          docker run --name=qdr $QDR_VOLUME $QDR_PORT -d $QDR_IMAGE
+      - name: Check if sg-bridge repository has same topic branch
+        id: bridge_branch
+        continue-on-error: true
+        run: |
+          echo "${GITHUB_REF#refs/heads/}"
+          git ls-remote --exit-code --heads https://github.com/infrawatch/sg-bridge.git "$(echo ${GITHUB_REF#refs/heads/})"
+      - name: Start sg-bridge from container image
+        if: steps.bridge_branch.outcome != 'success'
+        run: |
+          docker run --name=sgbridge --network host $BRIDGE_VOLUME -d \
+            $BRIDGE_IMAGE --amqp_url amqp://localhost:5666/$QDR_CHANNEL \
+            --gw_unix=$BRIDGE_SOCKET
+      - name: Start sg-bridge from same branch
+        if: steps.bridge_branch.outcome == 'success'
+        run: |
+          docker run --name=sgbridge --network host $BRIDGE_VOLUME -d -uroot \
+            -e GITHUB_REF -e BRIDGE_SOCKET -e QDR_CHANNEL -e OPSTOOLS_REPO \
             --workdir=$(dirname $BRIDGE_SOCKET) \
             $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_bridge.sh
       - name: Set Ceilometer pipelines to QDR output and restart notification agent
@@ -78,22 +178,15 @@ jobs:
           echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
           echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
           echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
-          cp ci/integration/metrics/*pipeline.yaml /etc/ceilometer/.
+          cp ci/integration/metrics/ceilometer/bridge/*pipeline.yaml /etc/ceilometer/.
           cat /etc/ceilometer/*
           sudo pip install pyngus
           sudo systemctl restart [email protected]
-      - name: Install collectd
-        run: |
-          sudo apt-get install collectd
-          sudo systemctl stop collectd && sudo systemctl disable collectd
-          sudo cp ci/integration/metrics/collectd.conf /etc/collectd/collectd.conf
-          sudo touch /var/log/collectd.log && sudo chmod a+rw /var/log/collectd.log
-          sudo collectd -C ci/integration/metrics/collectd.conf
       - name: Run sg-core to process metrics
         run: |
           docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
             --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
-            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_sg.sh
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/bridge/run_sg.sh
       - name: Run Prometheus to store metrics
         run: |
           docker run --name=prometheus -d --network host \
@@ -104,23 +197,94 @@ jobs:
           sleep 360
           echo "=========================== qdr =========================" && \
           docker exec qdr qdstat -b 127.0.0.1:5666 -a
+          docker logs qdr
           echo "========================= sg-core =======================" && \
           docker logs sgcore
-          echo "======================== collectd =======================" && \
-          cat /var/log/collectd.log
           echo "========================= ceilometer ====================" && \
           sudo journalctl -xu [email protected]
+          echo "======================== prometheus =====================" && \
+          docker logs prometheus
+      - name: Validate metrics processing
+        run: |
+          docker run --name=validate -uroot --network host \
+            --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh
+#-------------------------------------------------------------------------------
+  ceilometer-metrics-tcp:
+    name: "[metrics] transport: socket(tcp); handler: ceilometer-metrics; application: prometheus"
+    runs-on: ubuntu-22.04
+    env:
+      PROMETHEUS_IMAGE: prom/prometheus:latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Prepare environment
+        run: |
+          mkdir -p /opt/stack/
+          sudo setfacl -Rdm u::7,g::0,o:0 /opt/stack
+      - name: Prepare environment for mysql-server installation # https://stackoverflow.com/a/66026366
+        run: |
+          sudo apt-get -f install -o Dpkg::Options::="--force-overwrite"
+          sudo apt-get purge mysql\*
+          sudo rm -rf /var/lib/mysql
+          sudo rm -rf /etc/mysql
+          sudo dpkg -l | grep -i mysql
+          sudo apt-get clean
+      - name: Prepare environment for postgres-server installation
+        run: |
+          sudo apt remove postgresql-client-common
+          sudo apt install postgresql-client-common=238
+          sudo apt install postgresql-common
+          sudo python -m pip install --upgrade pip
+          sudo python -m pip install --upgrade virtualenv
+      - name: Install devstack
+        run: |
+          SOURCE=$(pwd)
+          git clone http://github.com/openstack/devstack /opt/stack/devstack
+          pushd /opt/stack/devstack
+          cp $SOURCE/ci/integration/metrics/local.conf .
+          sudo apt-get update
+          ./stack.sh
+          popd
+      - name: Set Ceilometer pipelines to TCP output and restart notification agent
+        run: |
+          sudo apt-get install -y crudini
+          echo addressing_mode="dynamic" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
+          echo pseudo_vhost=true | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
+          echo rpc_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
+          echo notify_address_prefix="" | crudini --merge /etc/ceilometer/ceilometer.conf oslo_messaging_amqp
+          cp ci/integration/metrics/ceilometer/tcp/*pipeline.yaml /etc/ceilometer/.
+          cat /etc/ceilometer/*
+          sudo pip install pyngus
+          sudo systemctl restart [email protected]
+      - name: Run sg-core to process metrics
+        run: |
+          docker run --name=sgcore -d -uroot --network host $BRIDGE_VOLUME -e OPSTOOLS_REPO \
+            --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/tcp/run_sg.sh
+      - name: Run Prometheus to store metrics
+        run: |
+          docker run --name=prometheus -d --network host \
+          --volume ${{ github.workspace }}/ci/integration/metrics/prometheus.yml:/etc/prometheus/prometheus.yml:ro \
+          $PROMETHEUS_IMAGE
+      - name: Debug output
+        run: |
+          sleep 360
           echo "========================= sg-core =======================" && \
+          docker logs sgcore
+          echo "========================= ceilometer ====================" && \
+          sudo journalctl -xu [email protected]
+          echo "======================== prometheus =====================" && \
           docker logs prometheus
       - name: Validate metrics processing
         run: |
           docker run --name=validate -uroot --network host \
             --volume ${{ github.workspace }}:$PROJECT_ROOT:z --workdir $PROJECT_ROOT \
-            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/run_validation.sh
+            $TEST_IMAGE bash $PROJECT_ROOT/ci/integration/metrics/ceilometer/run_validation.sh
 #-------------------------------------------------------------------------------
   logging:
     name: "[logging] handler: logs; application: elasticsearch, loki"
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     env:
       BRIDGE_SOCKET: /tmp/sg-bridge/test-socket
 
@@ -135,7 +299,7 @@ jobs:
       RSYSLOG_VOLUME: "--volume ${{ github.workspace }}/ci/service_configs/rsyslog/rsyslog_config.conf:/etc/rsyslog.d/integration.conf:z"
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       # start data store services
       - name: Start Elasticsearch service
         run: |
@@ -171,7 +335,7 @@ jobs:
             $RSYSLOG_IMAGE bash $PROJECT_ROOT/ci/integration/logging/run_rsyslog.sh
       - name: Wait for services to start successfuly
         run: |
-          timeout=180
+          timeout=240
           echo "======================= rsyslog ======================="
           rsyslog_wait=0
           while [[ $(docker exec qdr qdstat -b 127.0.0.1:5666 -a | grep rsyslog/logs | awk '{print $8}') -le 0 ]]

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -23,7 +23,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v3
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - uses: actions/checkout@v3
       #- name: download libraries
       #  run: go mod download
@@ -78,7 +78,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: actions/setup-go@v2
         with:
-          go-version: '1.18'
+          go-version: '1.19'
       - name: Verify image builds
         run: |
           docker build --tag infrawatch/sg-core:latest --file build/Dockerfile .

diff --git a/build.sh b/build.sh
@@ -14,6 +14,7 @@ base=$(pwd)
 GOCMD=${GOCMD:-"go"}
 PLUGIN_DIR=${PLUGIN_DIR:-"/tmp/plugins/"}
 CONTAINER_BUILD=${CONTAINER_BUILD:-false}
+BUILD_ARGS=${BUILD_ARGS:-''}
 
 PRODUCTION_BUILD=${PRODUCTION_BUILD:-false}
 if $PRODUCTION_BUILD; then
@@ -60,7 +61,7 @@ build_plugins() {
     search_list "$(basename $i)" OMIT_TRANSPORTS
     if [ $? -ne 1 ]; then
       echo "building $(basename $i).so"
-      $GOCMD build -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
+      $GOCMD build $BUILD_ARGS -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
     fi
   done
 
@@ -71,7 +72,7 @@ build_plugins() {
     search_list "$(basename $i)" OMIT_HANDLERS
     if [ $? -ne 1 ]; then
       echo "building $(basename $i).so"
-      $GOCMD build -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
+      $GOCMD build $BUILD_ARGS -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
     fi
   done
 
@@ -82,7 +83,7 @@ build_plugins() {
     search_list "$(basename $i)" OMIT_APPLICATIONS
     if [ $? -ne 1 ]; then
       echo "building $(basename $i).so"
-      $GOCMD build -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
+      $GOCMD build $BUILD_ARGS -o "$PLUGIN_DIR$(basename $i).so" -buildmode=plugin
     fi
   done
 }
@@ -92,9 +93,9 @@ build_core() {
   cd "$base"
   if $CONTAINER_BUILD; then
       echo "building sg-core for container"
-      $GOCMD build -o /tmp/sg-core cmd/*.go
+      $GOCMD build $BUILD_ARGS -o /tmp/sg-core cmd/*.go
   else
-      $GOCMD build -o sg-core cmd/*.go
+      $GOCMD build $BUILD_ARGS -o sg-core cmd/*.go
   fi
 }
 

diff --git a/build/Dockerfile b/build/Dockerfile
@@ -10,7 +10,7 @@ COPY . $D/
 COPY build/repos/opstools.repo /etc/yum.repos.d/opstools.repo
 
 RUN dnf install golang git qpid-proton-c-devel -y --setopt=tsflags=nodocs
-RUN go install golang.org/dl/go1.18@latest && /go/bin/go1.18 download && PRODUCTION_BUILD=false CONTAINER_BUILD=true GOCMD=/go/bin/go1.18 ./build.sh
+RUN go install golang.org/dl/go1.19@latest && /go/bin/go1.19 download && PRODUCTION_BUILD=false CONTAINER_BUILD=true GOCMD=/go/bin/go1.19 ./build.sh
 
 # --- end build, create smart gateway layer ---
 FROM registry.access.redhat.com/ubi8-minimal:latest

diff --git a/ci/integration/logging/run_sg.sh b/ci/integration/logging/run_sg.sh
@@ -13,11 +13,11 @@ dnf install -y git golang gcc make qpid-proton-c-devel
 export GOBIN=$GOPATH/bin
 export PATH=$PATH:$GOBIN
 
-go install golang.org/dl/go1.18@latest
-go1.18 download
+go install golang.org/dl/go1.19@latest
+go1.19 download
 
 # install sg-core and start sg-core
 mkdir -p /usr/lib64/sg-core
-PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.18 ./build.sh
+PLUGIN_DIR=/usr/lib64/sg-core/ GOCMD=go1.19 BUILD_ARGS=-buildvcs=false ./build.sh
 
 ./sg-core -config ./ci/integration/logging/sg_config.yaml
diff --git a/ci/integration/metrics/event_pipeline.yaml → ...ics/ceilometer/bridge/event_pipeline.yaml b/ci/integration/metrics/event_pipeline.yaml → ...ics/ceilometer/bridge/event_pipeline.yaml
diff --git a/ci/integration/metrics/pipeline.yaml → ...n/metrics/ceilometer/bridge/pipeline.yaml b/ci/integration/metrics/pipeline.yaml → ...n/metrics/ceilometer/bridge/pipeline.yaml