Merge branch 'develop'

Author: Thomas Franco

.github/actions/get_code_version/action.yml

@@ -18,4 +18,4 @@ runs:
       run: |
         V=$(sbt -no-colors --error "print version" | awk 'END{print $1}')
         echo "app_version=$V"
-        echo "app_version=$V" >> $GITHUB_OUTPUT
+        echo "app_version=$V" >> $GITHUB_OUTPUT

.github/workflows/build.docker.baselayer.yml

@@ -0,0 +1,72 @@
+name: build.docker.image.baselayer
+on:
+  schedule:
+    - cron: "19 2 * * *"
+  pull_request:
+    types:
+      - labeled
+  workflow_dispatch:
+  workflow_call:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    permissions:
+      actions: read
+      contents: read
+      packages: write
+    runs-on: [ linux ]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Generate full docker tags
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            name=${{ vars.SB_GHCR }}/cortex-baselayer
+          tags: |
+            type=raw,value=rolling
+          labels: |
+            org.opencontainers.image.title=cortex-baselayer
+            org.opencontainers.image.description=baselayer for Cortex final docker image
+            org.opencontainers.image.vendor=StrangeBee
+            org.opencontainers.image.version=rolling
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push docker image
+        uses: docker/build-push-action@v3
+        id: push
+        with:
+          context: .
+          file: baseLayer.Dockerfile
+          push: true
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}
+
+  notify:
+    needs: [ build ]
+    runs-on: [ ubuntu-latest ]
+    if: failure()
+    steps:
+      - name: Slack notification
+        uses: Gamesight/slack-workflow-status@master
+        with:
+          repo_token: ${{secrets.GITHUB_TOKEN}}
+          slack_webhook_url: ${{secrets.SLACK_WEBHOOK_URL}}
+          channel: "#ci-cortex"
+          name: Cortex baselayer build
+          include_commit_message: true
+          include_jobs: true

.github/workflows/build.docker.yml

@@ -1,10 +1,20 @@
-name: build.docker.image.dev
+name: build.docker.image.cortex
 on:
   pull_request:
     types:
       - labeled
   workflow_dispatch:
+    inputs:
+      is_prod:
+        description: "Publish docker image in dockerhub?"
+        type: boolean
+        default: false
   workflow_call:
+    inputs:
+      is_prod:
+        description: "Publish docker image in dockerhub?"
+        type: boolean
+        default: false
     outputs:
       image_id:
         description: "ImageId of the docker image"
@@ -19,7 +29,6 @@ on:
         description: "Version number of the Docker image"
         value: ${{ jobs.build.outputs.image_version }}
 
-
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -31,7 +40,7 @@ jobs:
       contents: write
     runs-on: [ self-hosted, linux, domain=sb ]
     outputs:
-      image_version: ${{ steps.get_version.outputs.version }}
+      image_version: ${{ steps.get_version.outputs.app_version }}
     steps:
       - uses: actions/checkout@v4
       - name: Set up Docker Buildx
@@ -54,6 +63,43 @@ jobs:
       image_metadata: ${{ steps.push.outputs.metadata }}
       image_version: ${{ needs.prepare.outputs.image_version }}
     steps:
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+      - name: echo version
+        id: image_details
+        run: |
+          '''
+          Following https://semver.org/#backusnaur-form-grammar-for-valid-semver-versions
+          SemVer should be used in EVERY project for standard usage of versions
+          '''
+          import os
+          import re
+
+          image_details = {}
+
+          if '+' in os.getenv('IMAGE_VERSION'):
+            image_details['build_version'] = os.getenv('IMAGE_VERSION').split('+')[1]
+
+          if '-' in os.getenv('IMAGE_VERSION'):
+            image_details['prerelease_version'] = os.getenv('IMAGE_VERSION').split('-')[1].split('+')[0]
+
+          image_details['core_version'] = os.getenv('IMAGE_VERSION').split('-')[0].split('+')[0]
+
+          image_details['major_version'] = os.getenv('IMAGE_VERSION').split('.')[0]
+          image_details['major_minor_version'] = re.search(r'(\d\.\d)', os.getenv('IMAGE_VERSION')).group()
+
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as gho:
+            print(f'image_details={image_details}', file=gho)
+        env:
+          IMAGE_VERSION: ${{ needs.prepare.outputs.image_version }}
+        shell: python
+
+      - name: simply print python results
+        run: |
+          echo ${{ steps.image_details.outputs.image_details }}
+
       - uses: actions/checkout@v4
 
       - name: Set up Docker Buildx
@@ -78,38 +124,54 @@ jobs:
       - name: Build packages
         run: sbt Docker/stage
 
-      # I'm not really at ease with these tags
-      # to me "latest" should be set manually, through a tag, for now
-      # but further, it should "calculate" it, regarding the latest Docker image version available
-      # for exemple if latest available is 3.2.0-1 and the sbt command returns 3.2.1-1,
-      # then latest should apply
-      # Moreover, the -1 is VERY important, because it increases with the number of builds:
-      # - if a Docker image exists with tag 3.2.0-1, it should NOT be overidden but a new
-      # 3.2.0-2 should be created, and the -1 cleaned up later 
-      #####
-      # TODO: work on tagging
-      #
-      #
+      - name: setup vault token
+        if: inputs.is_prod
+        run: echo "VAULT_TOKEN=$VAULT_TOKEN" >> $GITHUB_ENV
+
+      - name: Import prod secrets
+        if: inputs.is_prod
+        id: secrets_prod
+        uses: hashicorp/[email protected]
+        with:
+          url: https://vault.service.infra.sb:8200
+          token: ${{ env.VAULT_TOKEN }}
+          tlsSkipVerify: true
+          secrets: |
+            infra/data/ci/dockerhub username   | DOCKERHUB_USERNAME;
+            infra/data/ci/dockerhub token      | DOCKERHUB_TOKEN;
+
       - name: Generate full docker tags
         id: meta
         uses: docker/metadata-action@v4
         with:
           images: |
             name=${{ vars.SB_GHCR }}/cortex
+            name=thehiveproject/cortex,enable=${{ inputs.is_prod }}
           tags: |
-            type=raw,value=devel
+            type=raw,value=${{ fromJson(steps.image_details.outputs.image_details)['core_version'] }}
             type=raw,value=${{ needs.prepare.outputs.image_version }}
+            type=raw,value=${{ fromJson(steps.image_details.outputs.image_details)['major_version'] }}
+            type=raw,value=${{ fromJson(steps.image_details.outputs.image_details)['major_minor_version'] }}
           labels: |
             org.opencontainers.image.title=cortex
-            org.opencontainers.image.description=a Powerful Observable Analysis and Active Response Engine
+            org.opencontainers.image.description=A Powerful Observable Analysis and Active Response Engine
             org.opencontainers.image.vendor=StrangeBee
+            org.opencontainers.image.version=${{ needs.prepare.outputs.image_version }}
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Login to GitHub Container Registry
+        if: inputs.is_prod
+        uses: docker/login-action@v3
+        with:
+          username: ${{ steps.secrets_prod.outputs.DOCKERHUB_USERNAME }}
+          password: ${{ steps.secrets_prod.outputs.DOCKERHUB_TOKEN }}
+
       - name: Build and push docker image
         uses: docker/build-push-action@v3
         id: push
@@ -118,3 +180,4 @@ jobs:
           push: true
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/deploy.nomad.yml

@@ -5,8 +5,7 @@ on:
       docker_image_version:
         type: string
         required: false
-        description: "Docker image version to deploy (default 'devel')"
-        default: 'devel'
+        description: "Docker image version to deploy"
       from_docker_hub:
         type: boolean
         required: false
@@ -20,7 +19,7 @@ on:
 # pull_request = pr-XXX
 # manual = vXXX
 env:
-  deployment_version: ${{ github.event_name == 'pull_request' && format('{0}-{1}', 'pr', github.event.number) || format('{0}{1}', (inputs.docker_image_version != 'devel' && 'v' || ''), inputs.docker_image_version) }}
+  deployment_version: ${{ github.event_name == 'pull_request' && format('{0}-{1}', 'pr', github.event.number) || format('{0}{1}', 'v', inputs.docker_image_version) }}
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -77,7 +76,11 @@ jobs:
       - name: Deploy job using Nomad Pack
         id: run
         # We pass two different version variables: one for Docker pull, the other for Nomad services
-        run: nomad-pack run -var from_docker_hub=${{ inputs.from_docker_hub }} -var docker_image_version=${{ inputs.docker_image_version || env.deployment_version }} -var service_version=${{ needs.prepare.outputs.expected_deployment_version }} ./deployment/nomad/packs/cortex
+        run: |
+          nomad-pack run -var from_docker_hub=${{ inputs.from_docker_hub }} \
+                         -var docker_image="${{ vars.SB_GHCR }}/cortex" \
+                         -var docker_image_version=${{ inputs.docker_image_version || env.deployment_version }} \
+                         -var service_version=${{ needs.prepare.outputs.expected_deployment_version }} ./deployment/nomad/packs/cortex
         env:
           NOMAD_ADDR: "http://10.30.4.180:4646"
           NOMAD_TOKEN: "${{ env.NOMAD_TOKEN }}"

.github/workflows/setup.fixtures.yml

@@ -7,16 +7,14 @@
       inputs:
         expected_deployment_version:
           type: string
-          required: false
-          description: "Docker image version to deploy (default 'latest')"
-          default: "latest"
+          required: true
+          description: "Docker image version to deploy"
     workflow_call:
       inputs:
         expected_deployment_version:
           type: string
-          required: false
-          description: "Docker image version to deploy (default 'latest')"
-          default: "latest"
+          required: true
+          description: "Docker image version to deploy"
 
   jobs:
     setup:

app/org/thp/cortex/services/K8sJobRunnerSrv.scala

@@ -1,14 +1,15 @@
 package org.thp.cortex.services
 
 import akka.actor.ActorSystem
-import io.fabric8.kubernetes.api.model.PersistentVolumeClaimVolumeSourceBuilder
-import io.fabric8.kubernetes.api.model.batch.{JobBuilder => KJobBuilder}
+import io.fabric8.kubernetes.api.model.{PersistentVolumeClaimVolumeSourceBuilder, StatusDetails}
+import io.fabric8.kubernetes.api.model.batch.v1.{JobBuilder => KJobBuilder}
 import io.fabric8.kubernetes.client.DefaultKubernetesClient
 import org.thp.cortex.models._
 import org.thp.cortex.util.FunctionalCondition._
 import play.api.{Configuration, Logger}
 
 import java.nio.file._
+import java.util
 import javax.inject.{Inject, Singleton}
 import scala.concurrent.duration.{DurationInt, FiniteDuration}
 import scala.jdk.CollectionConverters._
@@ -113,7 +114,7 @@ class K8sJobRunnerSrv(
     .build()
 
     val execution = Try {
-      val created_kjob = client.batch().jobs().create(kjob)
+      val created_kjob = client.batch().v1().jobs().create(kjob)
       val created_env = created_kjob
         .getSpec.getTemplate.getSpec.getContainers.get(0)
         .getEnv.asScala
@@ -123,7 +124,7 @@ class K8sJobRunnerSrv(
         s"  image  : $dockerImage\n" +
         s"  mount  : pvc $persistentVolumeClaimName subdir $relativeJobDirectory as /job" +
         created_env.map(ev => s"\n  env    : ${ev.getName} = ${ev.getValue}").mkString)
-      val ended_kjob = client.batch().jobs().withLabel("cortex-job-id", job.id)
+      val ended_kjob = client.batch().v1().jobs().withLabel("cortex-job-id", job.id)
         .waitUntilCondition(x => Option(x).flatMap(j =>
           Option(j.getStatus).flatMap(s =>
             Some(s.getConditions.asScala.map(_.getType).exists(t =>
@@ -139,8 +140,8 @@ class K8sJobRunnerSrv(
     }
     // let's find the job by the attribute we know is fundamentally
     // unique, rather than one constructed from it
-    val deleted = client.batch().jobs().withLabel("cortex-job-id", job.id).delete()
-    if(deleted) {
+    val deleted: util.List[StatusDetails] = client.batch().v1().jobs().withLabel("cortex-job-id", job.id).delete()
+    if(!deleted.isEmpty) {
       logger.info(s"Deleted Kubernetes Job for job ${job.id}")
     } else {
       logger.info(s"While trying to delete Kubernetes Job for ${job.id}, the job was not found; this is OK")

baseLayer.Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.12.9-bookworm
+
+LABEL MAINTAINER="TheHive Project <[email protected]>" repository="https://github.com/TheHive-Project/TheHive"
+ENV JAVA_HOME=/usr/lib/jvm/java-11-amazon-corretto
+RUN apt update && \
+    apt upgrade -y && \
+    apt install -y curl \
+                   gnupg && \
+    curl -fL https://apt.corretto.aws/corretto.key | gpg --dearmor -o /usr/share/keyrings/corretto.gpg && \
+    echo 'deb [signed-by=/usr/share/keyrings/corretto.gpg] https://apt.corretto.aws stable main' > /etc/apt/sources.list.d/corretto.list && \
+    apt update && \
+    apt install -y java-11-amazon-corretto-jdk && \
+    curl -fsSL https://download.docker.com/linux/debian/gpg -o /usr/share/keyrings/docker.asc && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list && \
+    apt update && \
+    apt install -y docker-ce \
+                   docker-ce-cli \
+                   containerd.io \
+                   docker-ce-rootless-extras \
+                   uidmap \
+                   iproute2 \
+                   fuse-overlayfs && \
+    rm -rf /var/lib/apt/lists/* && \
+    apt autoclean -y -q && \
+    apt autoremove -y -q
+CMD ["bash"]

build.sbt

@@ -1,4 +1,4 @@
-import Common._
+import Common.*
 
 ThisBuild / scalaVersion := Dependencies.scalaVersion
 ThisBuild / evictionErrorLevel := util.Level.Warn