From 688144d7b349be6177203d926fa948ed83ac6771 Mon Sep 17 00:00:00 2001
From: Milan Mertens <m.mertens@metaways.de>
Date: Thu, 7 Mar 2024 10:03:52 +0100
Subject: [PATCH 1/9] ci(npm-install) allow additional arguments

---
 ci/gitlab-ci/lib.yml             | 5 +++--
 ci/gitlab-ci/lib/scripts/test.sh | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/ci/gitlab-ci/lib.yml b/ci/gitlab-ci/lib.yml
index ff09080fe3..ac0c407823 100644
--- a/ci/gitlab-ci/lib.yml
+++ b/ci/gitlab-ci/lib.yml
@@ -990,8 +990,9 @@
       }
       
       test_npm_install() {
+          additional_npm_args="$1"
           log "trying to use cache..."
-          package_shrinkwrap_hash=$(cd ${TINE20ROOT}/tine20/Tinebase/js; sha1sum npm-shrinkwrap.json package.json | sha1sum | cut -d ' ' -f 1)
+          package_shrinkwrap_hash=$(cd ${TINE20ROOT}/tine20/Tinebase/js; echo $(sha1sum npm-shrinkwrap.json package.json)"$additional_npm_args" | sha1sum | cut -d ' ' -f 1)
           export NODE_MODULE_CACHE_DIR=${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}/tine20/npm-cache/v1/${package_shrinkwrap_hash}
       
           mkdir -p $(dirname $NODE_MODULE_CACHE_DIR)
@@ -1009,7 +1010,7 @@
       
           log "installing npm ..."
           pushd ${TINE20ROOT}/tine20/Tinebase/js
-          npm --no-optional install
+          npm --no-optional install $additional_npm_args
           popd
       
           if [ ! -d $NODE_MODULE_CACHE_DIR ]; then
diff --git a/ci/gitlab-ci/lib/scripts/test.sh b/ci/gitlab-ci/lib/scripts/test.sh
index 2ed16c9ac2..95c7caff55 100644
--- a/ci/gitlab-ci/lib/scripts/test.sh
+++ b/ci/gitlab-ci/lib/scripts/test.sh
@@ -106,8 +106,9 @@ test_composer_install() {
 }
 
 test_npm_install() {
+    additional_npm_args="$1"
     log "trying to use cache..."
-    package_shrinkwrap_hash=$(cd ${TINE20ROOT}/tine20/Tinebase/js; sha1sum npm-shrinkwrap.json package.json | sha1sum | cut -d ' ' -f 1)
+    package_shrinkwrap_hash=$(cd ${TINE20ROOT}/tine20/Tinebase/js; echo $(sha1sum npm-shrinkwrap.json package.json)"$additional_npm_args" | sha1sum | cut -d ' ' -f 1)
     export NODE_MODULE_CACHE_DIR=${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}/tine20/npm-cache/v1/${package_shrinkwrap_hash}
 
     mkdir -p $(dirname $NODE_MODULE_CACHE_DIR)
@@ -125,7 +126,7 @@ test_npm_install() {
 
     log "installing npm ..."
     pushd ${TINE20ROOT}/tine20/Tinebase/js
-    npm --no-optional install
+    npm --no-optional install $additional_npm_args
     popd
 
     if [ ! -d $NODE_MODULE_CACHE_DIR ]; then

From 85d30682015b63c12928ed5351256b42d122680c Mon Sep 17 00:00:00 2001
From: Milan Mertens <m.mertens@metaways.de>
Date: Tue, 27 Feb 2024 15:17:20 +0100
Subject: [PATCH 2/9] ci(lib.yaml): make lib functions available in entrypoints

---
 ci/gitlab-ci/lib.yml                          | 1062 ++++++++++++++++-
 .../lib/entry_points/tine_as_a_service.sh     |    1 -
 ci/gitlab-ci/lib/scripts/matrix.sh            |    1 -
 ci/gitlab-ci/makefile                         |   16 +-
 4 files changed, 1068 insertions(+), 12 deletions(-)

diff --git a/ci/gitlab-ci/lib.yml b/ci/gitlab-ci/lib.yml
index ac0c407823..1135012f72 100644
--- a/ci/gitlab-ci/lib.yml
+++ b/ci/gitlab-ci/lib.yml
@@ -415,7 +415,6 @@
       }
 
       # lib/scripts/matrix.sh
-      #!/bin/sh
       matrix_send_message() {
           roomid=$1
           message=$2
@@ -1060,12 +1059,1064 @@
       }
 
   entry_points:
+    # lib/entry_points/tine_as_a_service.sh
     tine_as_a_service:
-      - bash
+      - 
       - -c
       - |
-        # lib/entry_points/tine_as_a_service.sh
-        #!/bin/bash
+        # lib/scripts/cache_cleanup.sh
+        # requres gnu date - busybox date wont work
+        function cache_cleanup_atomic_dir_cache () {
+            MAX_AGE_USED_CACHE=$1
+            MAX_AGE_CACHE=$2
+        
+            dirs=$(find . -type d -path './*' -prune -print)
+        
+            threshold_used=$(( $(date +%s) -  60 * 60 * 24 * $MAX_AGE_USED_CACHE ))
+            threshold=$(( $(date +%s) -  60 * 60 * 24 * $MAX_AGE_CACHE ))
+        
+            for dir in $dirs; do
+                if [ -f $dir-lastused ]; then
+                    lastused=$(date -d $(cat $dir-lastused) +%s)
+        
+                    if [ $lastused -gt $threshold_used ]; then
+                        continue
+                    fi
+        
+                    rm -f $dir-lastused
+                fi
+        
+                modified=$(stat -c %Y $dir)
+                if [ $modified -gt $threshold ]; then
+                    continue
+                fi
+        
+                rm -rf $dir
+            done
+        }
+        
+        function cache_cleanup_vendor_dir_cache () {
+            MAX_AGE_USED_CACHE=14
+            MAX_AGE_CACHE=2
+        
+            cd ${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}/tine20/composer-cache/v1/
+        
+            cache_cleanup_atomic_dir_cache $MAX_AGE_USED_CACHE $MAX_AGE_CACHE
+        }
+        
+        function cache_cleanup_npm_dir_cache () {
+            MAX_AGE_USED_CACHE=14
+            MAX_AGE_CACHE=2
+        
+            cd ${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}/tine20/npm-cache/v1/
+        
+            cache_cleanup_atomic_dir_cache $MAX_AGE_USED_CACHE $MAX_AGE_CACHE
+        }
+        
+        function cache_cleanup_phpstan_cache () {
+            MAX_AGE_USED_CACHE=31
+            MAX_AGE_CACHE=7
+        
+            cd ${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}
+        
+            dirs=$(find . -type d -path './*' -prune -print)
+            for dir in $dirs; do
+                if ! [ -d $dir/phpstan-cache/v2/ ]; then
+                    continue
+                fi
+        
+                cd $dir/phpstan-cache/v2/
+                cache_cleanup_atomic_dir_cache $MAX_AGE_USED_CACHE $MAX_AGE_CACHE
+            done
+        }
+
+        # lib/scripts/docker_build_image.sh
+        docker_build_image() {
+            local target=$1;
+            local hash=$2
+        
+            echo "building image: target: ${target}; tag: ${hash}";
+        
+            if [[ "$DOCKER_IMAGE_CACHE" == "false" ]] || ! docker_registry_image_exists ${target} ${hash}; then
+                echo "building image ...";
+        
+                local LAYER_CACHE_IMAGE="${REGISTRY}/${TARGET}:$(echo ${CI_COMMIT_REF_NAME} | sed sI/I-Ig)-${PHP_VERSION}${IMAGE_TAG_PLATFORM_POSTFIX}"
+                local MAJOR_LAYER_CACHE_IMAGE="${REGISTRY}/${TARGET}:$(echo ${MAJOR_COMMIT_REF_NAME} | sed sI/I-Ig)-${PHP_VERSION}${IMAGE_TAG_PLATFORM_POSTFIX}"
+        
+                cd ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20;
+        
+                cp $DOCKER_GIT_CONFIG ./ci/dockerimage/.gitconfig
+                ./ci/dockerimage/make.sh -u -p -i "${REGISTRY}/${target}:${hash}" -c "${LAYER_CACHE_IMAGE}" -c "${MAJOR_LAYER_CACHE_IMAGE}" ${target};
+            else
+                echo "image exists ...";
+            fi;
+        }
+
+        # lib/scripts/docker_hub.sh
+        docker_hub_deploy () {
+            set -e
+            name=$1
+            dockerhubname=$2
+            dockerhubtag=$3
+        
+            docker login -u "${DOCKERHUB_USER}" -p "${DOCKERHUB_TOKEN}" "docker.io"
+        
+            FROM_IMAGE="${REGISTRY}/${name}-commit:${IMAGE_TAG}"
+            DESTINATION_IMAGE="docker.io/tinegroupware/${dockerhubname}:${dockerhubtag}"
+        
+            docker pull "${FROM_IMAGE}"
+            docker tag "${FROM_IMAGE}" "${DESTINATION_IMAGE}"
+            docker push "${DESTINATION_IMAGE}"
+        }
+        
+        docker_hub_deploy_with_tag () {
+            docker_hub_deploy $1 $2 $CI_COMMIT_TAG
+            docker_hub_deploy $1 $2 $(echo $CI_COMMIT_TAG | cut -d '.' -f 1)
+        }
+
+        # lib/scripts/docker_image_hash.sh
+        _longest_common_prefix() {
+            declare -a names;
+            declare -a parts;
+            declare i=0;
+        
+            names=("$@");
+            name="$1";
+            while x=$(dirname "$name"); [ "$x" != "/" ] && [ "$x" != "." ]; do
+                parts[$i]="$x";
+                i=$(($i + 1));
+                name="$x";
+            done;
+        
+            for prefix in "${parts[@]}" /; do
+                for name in "${names[@]}"; do
+                if [ "${name#$prefix/}" = "${name}" ]; then
+                    continue 2;
+                fi;
+                done;
+                echo "$prefix";
+                return;
+            done;
+            echo ".";
+        }
+        
+        _path_without_prefix() {
+            local prefix="$1/";
+            shift;
+            local arg;
+            for arg in "$@"; do
+                echo "${arg#$prefix}";
+            done;
+        }
+        
+        file_hashes() {
+            local pattern=$@;
+        
+            local lcp=$(_longest_common_prefix $pattern);
+            local pwp=$(_path_without_prefix $lcp $pattern);
+        
+            local pwd=$(pwd);
+            cd $lcp;
+        
+            find $pwp -type f -exec sha256sum {} +;
+            if [ ${PIPESTATUS[0]} != 0 ]; then
+                exit 1;
+            fi;
+        
+            cd $pwd;
+        }
+        
+        _base_image_hash() {
+            cd ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20;
+            local fh=$(file_hashes ci/dockerimage/base.Dockerfile ci/dockerimage/confd/ ci/dockerimage/scripts/ ci/dockerimage/supervisor.d/ etc/nginx etc/tine20/config.inc.php.tmpl);
+                
+            echo $fh $TINE20ROOT $PHP_VERSION $IMAGE_TAG_PLATFORM_POSTFIX | sha256sum | head -c 32;
+        }
+        
+        _dependency_image_hash() {
+            cd ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20;
+            local fh=$(file_hashes ci/dockerimage/dependency.Dockerfile tine20/library tine20/composer.json tine20/composer.lock tine20/Tinebase/js/package.json tine20/Tinebase/js/npm-shrinkwrap.json scripts/packaging/composer/composerLockRewrite.php);
+        
+            echo $fh $TINE20ROOT $(_base_image_hash) | sha256sum | head -c 32;
+        }
+        
+        _test_dependency_image_hash() {
+            cd ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20;
+            local fh=$(file_hashes ci/dockerimage/test-dependency.Dockerfile ci/dockerimage/supervisor.d/webpack.ini etc phpstan.neon phpstan-baseline.neon);
+        
+            echo $fh $TINE20ROOT $(_dependency_image_hash) | sha256sum | head -c 32;
+        }
+        
+        docker_image_hash() {
+            case $1 in
+                base)
+                    _base_image_hash;
+                    ;;
+                dependency)
+                    _dependency_image_hash;
+                    ;;
+                test-dependency)
+                    _test_dependency_image_hash;
+                    ;;
+            esac
+        }
+
+        # lib/scripts/docker_layer_cache.sh
+        docker_layer_cache_populat_with_hash_image() {
+            target=$1;
+            hash=$2;
+        
+            docker_registry_rename_remote $REGISTRY_USER $REGISTRY_PASSWORD $REGISTRY $target $hash $target $(echo $CI_COMMIT_REF_NAME | sed sI/I-Ig)-${PHP_VERSION}${IMAGE_TAG_PLATFORM_POSTFIX};
+        }
+
+        # lib/scripts/docker_registry.sh
+        docker_registry_image_exists() {
+            local image=$1;
+            local tag=$2;
+            curl -s -f --user $REGISTRY_USER:$REGISTRY_PASSWORD -H "accept: application/vnd.docker.distribution.manifest.v2+json" "https://${REGISTRY}/v2/${image}/manifests/${tag}" > /dev/null;
+        }
+        
+        docker_registry_rename_remote() {
+            local user=$1;
+            local password=$2;
+            local registry=$3;
+            local old_repo=$4;
+            local old_tag=$5;
+            local new_repo=$6;
+            local new_tag=$7;
+        
+            echo "$0 retagging $registry $old_repo/$old_tag to $new_repo/$new_tag";
+        
+            if ! curl -s -f --user $user:$password -H "accept: application/vnd.docker.distribution.manifest.v2+json" https://$registry/v2/$old_repo/manifests/$old_tag > /dev/null; then
+                curl -s --user $user:$password -H "accept: application/vnd.docker.distribution.manifest.v2+json" https://$registry/v2/$old_repo/manifests/$old_tag;
+                exit 1;
+            fi;
+        
+            manifest=$(curl -s -f -X GET --user $user:$password -H "accept: application/vnd.docker.distribution.manifest.v2+json"  https://$registry/v2/$old_repo/manifests/$old_tag);
+        
+            for digest in $(echo $manifest | jq -r '.layers[].digest'); do
+                curl -s -f -X POST --user $user:$password "https://$registry/v2/$new_repo/blobs/uploads/?mount=$digest&from=$old_repo";
+            done;
+        
+            curl -s -f -X POST --user $user:$password "https://$registry/v2/$new_repo/blobs/uploads/?mount=$(echo $manifest | jq -r '.config.digest')&from=$old_repo";
+            curl -s -f -X PUT --user $user:$password -H "Content-Type: application/vnd.docker.distribution.manifest.v2+json" --data "$manifest" https://$registry/v2/$new_repo/manifests/$new_tag;
+        }
+        
+        docker_registry_use_hash_image_as_commit_image () {
+            source=$1;
+            target=$2;
+            hash=$3;
+        
+            docker_registry_rename_remote $REGISTRY_USER $REGISTRY_PASSWORD $REGISTRY $source $hash ${target}-commit ${IMAGE_TAG};
+        }
+        
+        docker_registry_login () {
+            registry="$1"
+            username="$2"
+            password="$3"
+        
+            for i in {1..6}; do
+                if docker login "$registry" --username "$username" --password "$password"; then
+                    return 0
+                fi
+        
+                echo "($i) docker login failed, retrying it in 5 second ..."
+                curl https://${REGISTRY}/fail-${CI_PIPELINE_ID}-${CI_JOB_ID} # create a marker in the log if login fails
+                sleep 5
+            done
+        
+            echo "docker login failed, aborting ..."
+            return 1
+        }
+        
+        docker_registry_release_image() {
+            name="$1"
+            desitination="$2"
+            latest="$3"
+        
+            from="${REGISTRY}/${name}-commit:${IMAGE_TAG}"
+        
+            if [ -z "$CI_COMMIT_TAG" ]; then
+                echo "pushing nightly"
+                docker_registry_push "${from}" "${desitination}:dev-$(git describe --tags)"
+                return
+            fi
+        
+            if [ "$latest" == "true" ]; then
+                docker_registry_push "${from}" "${desitination}:latest"
+            fi
+        
+            docker_registry_push "${from}" "${desitination}:${CI_COMMIT_TAG}"
+            docker_registry_push "${from}" "${desitination}:$(echo ${CI_COMMIT_TAG} | cut -d '.' -f 1)"
+        }
+        
+        docker_registry_push() {
+            from="$1"
+            to="$2"
+        
+            docker pull "${from}"
+            docker tag "${from}" "${to}"
+            docker push "${to}"
+        }
+
+        # lib/scripts/git_repo.sh
+        git_repo_clone () {
+            git clone ${CI_REPOSITORY_URL} --branch ${CI_COMMIT_REF_NAME} --depth 1 --single-branch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20
+        }
+        
+        git_repo_clone_cached () {
+            targetDir=${1:-$CI_PROJECT_DIR}
+        
+            cacheRepoPath=${CI_CUSTOM_CACHE_DIR}/git-cache/v2/$(echo "${CI_REPOSITORY_URL}" | cut -d '@' -f2)/$CI_CONCURRENT_ID
+            echo $cacheRepoPath
+            
+            mkdir -p $cacheRepoPath
+        
+            pushd $cacheRepoPath
+            if ! git rev-parse --is-inside-work-tree 2> /dev/null; then
+                log "not a git repo. Initialzing new bare git repo ..."
+                git init --bare
+            fi
+        
+            log "setting up remote"
+            if git remote | grep origin > /dev/null; then
+                git remote rm origin
+            fi
+        
+            git remote add origin "${CI_REPOSITORY_URL}"
+        
+            log "fetching from gitlab" 
+            git fetch --force origin '*:*'
+            popd
+        
+        
+            log "cloning into working dir"
+            git clone $cacheRepoPath --branch "${CI_COMMIT_REF_NAME}" --local --no-hardlinks "${targetDir}"
+            pushd ${targetDir}
+            # switch orign from cache to gitlab
+            git remote rm origin
+            git remote add origin "${CI_REPOSITORY_URL}"
+            # fetch new origin again
+            git fetch
+            # setup branch upstream again
+            git branch --set-upstream-to="origin/${CI_COMMIT_REF_NAME}" "${CI_COMMIT_REF_NAME}"
+            popd
+        }
+
+        # lib/scripts/github.sh
+        github_get_release_by_tag() {
+            owner=$1
+            repo=$2
+            tag=$3
+        
+            curl -s \
+                -H "accept: application/vnd.github.v3+json" \
+                "https://api.github.com/repos/$owner/$repo/releases/tags/$tag"
+        }
+        
+        github_create_release() {
+            owner=$1
+            repo=$2
+            tag=$3
+            user=$4
+            token=$5
+        
+            body="$(github_create_release_body $owner $repo $tag | jq -Rs .)"
+        
+            curl -s \
+                -X POST \
+                -u "$user:$token" \
+                -H "accept: application/vnd.github.v3+json" \
+                "https://api.github.com/repos/$owner/$repo/releases" \
+                -d '{"tag_name":"'"$tag"'", "body":'"$body"'}'
+        }
+        
+        github_create_release_body() {
+            owner=$1
+            repo=$2
+            tag=$3
+        
+            previous_tag=$(github_get_latest_release_tag_name "$owner" "$repo")
+        
+            git fetch origin "$previous_tag"
+        
+            if ! git log "$tag...$previous_tag" 1> /dev/null; then
+                git fetch origin --unshallow --quiet
+            fi
+        
+            repo_release_notes "$tag" "$previous_tag"
+        }
+        
+        github_release_add_asset() {
+            release_json=$1
+            name=$2
+            path_to_asset=$3
+            user=$4
+            token=$5
+        
+            upload_url=$(echo $release_json | jq -r '.upload_url')
+            upload_url="${upload_url%\{*}"
+        
+            curl -s \
+                -X POST \
+                -u "$user:$token" \
+                -T "$path_to_asset" \
+                -H "accept: application/vnd.github.v3+json" \
+                -H "content-type: $(file -b --mime-type $path_to_asset)" \
+                "$upload_url?name=$name.tar.bz2"
+        }
+        
+        github_get_latest_release_tag_name() {
+            owner=$1
+            repo=$2
+        
+            curl https://api.github.com/repos/$1/$2/releases | jq -r '.[0].tag_name'
+        }
+
+        # lib/scripts/matrix.sh
+        matrix_send_message() {
+            roomid=$1
+            message=$2
+        
+            if test -z "$MATRIX_SERVER"; then
+                echo "MATRIX_SERVER needs to be set!"
+                return 1
+            fi
+        
+            if test -z "$MATRIX_ACCESS_TOKEN"; then
+                if test -z "$MATRIX_USERNAME" || test -z "$MATRIX_PASSWORD"; then
+                    echo "Either MATRIX_ACCESS_TOKEN or MATRIX_USERNAME and MATRIX_PASSWORD needs to be set!"
+                    return 1
+                fi
+            
+                response=$(curl -s -XPOST -d '{"type":"m.login.password", "user":"'"$MATRIX_USERNAME"'", "password":"'"$MATRIX_PASSWORD"'"}' "https://$MATRIX_SERVER/_matrix/client/r0/login")
+                MATRIX_ACCESS_TOKEN=$(echo "$response" | jq -r '.access_token')
+                
+            fi
+        
+            curl -XPOST -d '{}' "https://$MATRIX_SERVER/_matrix/client/r0/rooms/$roomid/join?access_token=$MATRIX_ACCESS_TOKEN"
+        
+            curl -XPOST -d '{"msgtype":"m.text", "body":"'"$message"'"}' "https://$MATRIX_SERVER/_matrix/client/r0/rooms/$roomid/send/m.room.message?access_token=$MATRIX_ACCESS_TOKEN"
+        }
+
+
+        # lib/scripts/merge.sh
+        merge_merge_upwards () {
+            if ! ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/scripts/git/merge_helper.sh MergeUpwards "$1" "$2" "customers"; then
+                matrix_send_message "$MATRIX_ROOM" "🔴 Auto merging $1 into $2 failed in $CI_PIPELINE_NAME $CI_JOB_URL."
+                return 1
+            fi
+        }
+        
+        merge_update_custom_app () {
+            ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/scripts/git/merge_helper.sh UpdateCustomApp "$1" "$2" || true
+        }
+        
+        merge_merge_mirror () {
+            source_remote="$1"
+            source_branch="$2"
+            destination_remote="$3"
+            destination_branch="$4"
+        
+            git fetch "$source_remote" "$source_branch" || return 1
+            git fetch "$destination_remote" "$destination_branch" || return 1
+        
+            if git rev-parse --quiet --verify $destination_branch > /dev/null; then
+                git checkout "$destination_branch"
+                git reset --hard "$destination_remote/$destination_branch"
+            else
+                git checkout --track "$destination_remote/$destination_branch" || return 1
+            fi
+            
+            echo "git mergeing $source_remote/$source_branch into $destination_remote/$destination_branch ..."
+        
+            if ! git merge "$source_remote/$source_branch"; then
+        
+                if ! php ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/scripts/git/repairMerge.php "$source_remote/$source_branch" "$destination_branch"; then
+                    echo "merging $source_remote/$source_branch into $destination_remote/$destination_branch failed"
+                    return 1
+                fi
+            fi
+            
+            git push "$destination_remote" "$destination_branch"
+        }
+        
+        merge_trigger_next () {
+            MERGE_MAP=${MERGE_MAP:-"{}"}
+            
+            if ! echo $MERGE_MAP | jq --arg ref $CI_COMMIT_REF_NAME -e '.[$ref]' > /dev/null; then
+                echo "nothing to trigger"
+                return
+            fi
+        
+            for i in $(echo $MERGE_MAP | jq -c --arg ref $CI_COMMIT_REF_NAME '.[$ref][]'); do
+                ref=$(echo $i | jq -r '.ref')
+                var=$(echo $i | jq -r '.var')
+        
+                echo "trigger $ref with $var:"
+        
+                curl -X POST -F token=$MERGE_TRIGGER_TOKEN \
+                    -F ref=$ref \
+                    -F "variables[$var]=true" \
+                    -F "variables[DOCKER_BUILD_SOURCE]=true" \
+                    -F "variables[SEND_PIPELINE_STATUS]=true" \
+                    "$CI_API_V4_URL/projects/$CI_PROJECT_ID/trigger/pipeline" > /dev/null
+            done
+        }
+
+        # lib/scripts/packaging.sh
+        packaging_build_packages() {
+            version=$1
+            release=$2
+        
+            CI_COMMIT_REF_NAME_ESCAPED=$(echo ${CI_COMMIT_REF_NAME} | sed sI/I-Ig)
+            MAJOR_COMMIT_REF_NAME_ESCAPED=$(echo ${MAJOR_COMMIT_REF_NAME} | sed sI/I-Ig)
+        
+            CACHE_IMAGE="${REGISTRY}/packages:${CI_COMMIT_REF_NAME_ESCAPED}-${PHP_VERSION}${IMAGE_TAG_PLATFORM_POSTFIX}"
+            MAJOR_CACHE_IMAGE="${REGISTRY}/packages:${MAJOR_COMMIT_REF_NAME_ESCAPED}-${PHP_VERSION}${IMAGE_TAG_PLATFORM_POSTFIX}"
+        
+            if echo "$CI_COMMIT_TAG" | grep '/'; then
+                echo "Error: CI_COMMIT_TAG must not contain a /"
+                exit 1
+            fi
+        
+            # config via env
+            export PHP_VERSION=${PHP_VERSION}
+            export BASE_IMAGE="${REGISTRY}/base-commit:${IMAGE_TAG}"
+            export DEPENDENCY_IMAGE="${REGISTRY}/dependency-commit:${IMAGE_TAG}"
+            export SOURCE_IMAGE="${REGISTRY}/source-commit:${IMAGE_TAG}"
+            export BUILD_IMAGE="${REGISTRY}/build-commit:${IMAGE_TAG}"
+            export BUILT_IMAGE="${REGISTRY}/built-commit:${IMAGE_TAG}"
+            export REVISION=0
+            export CODENAME="${CODENAME}"
+            export VERSION=$version
+            export RELEASE=$release
+        
+            cd ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20
+            # create archives
+            if ! ./ci/dockerimage/make.sh -o "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/packages.tar" -c "${CACHE_IMAGE}" -c "${MAJOR_CACHE_IMAGE}" packages; then
+                return 1
+            fi
+        
+            # add current.map
+            if ! echo "$version" | grep "nightly" && ! echo "$version" | grep "weekly" ; then
+                echo "currentPackage ${RELEASE}/tine20-allinone_${RELEASE}.tar.bz2" >> current.map
+                tar -rf "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/packages.tar" current.map
+            else
+                echo "nightly, do not set curren.map"
+            fi
+        }
+        
+        packaging_extract_all_package_tar() {
+            cd "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/"
+            tar -xf "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/packages.tar"
+        }
+        
+        packaging_push_packages_to_gitlab() {
+            version=$1
+            release=$2
+        
+            customer=$(release_determin_customer)
+        
+            curl -S -s \
+                --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
+                --upload-file "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/packages.tar" \
+                "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/${version}/all.tar"
+        
+            echo "published packages to ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/${version}/all.tar"
+        
+            cd "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/${release}/"
+        
+            for f in *; do
+                curl -S -s \
+                --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
+                --upload-file "$f" \
+                "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/${version}/$f"
+            done
+        
+            echo ""
+            echo "published packages to ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/${version}/all.tar"
+            echo ""
+        }
+        
+        packaging_gitlab_set_ci_id_link() {
+            version=$1
+            customer=$(release_determin_customer)
+        
+            if ! curl -S -s \
+                --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
+                -XPUT --data "${version}" \
+                "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/links/${CI_PIPELINE_ID}"
+            then
+                return 1
+            fi
+        }
+        
+        packaging_gitlab_get_version_for_pipeline_id() {
+            customer=$1
+        
+            if ! curl \
+                --fail \
+                --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
+                "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/links/${CI_PIPELINE_ID}"
+            then
+                return 1
+            fi
+        }
+        
+        packaging_gitlab_set_current_link() {
+            customer=$(release_determin_customer)
+            version=${CI_COMMIT_TAG:-$(packaging_gitlab_get_version_for_pipeline_id ${customer})}
+        
+            if echo "$version" | grep "nightly"; then
+                echo "skip setting current link for nightly packages: $version"
+                return 0
+            fi
+        
+            curl \
+                --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \
+                -XPUT --data "${version}" \
+                "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/links/current"
+        
+            matrix_send_message $MATRIX_ROOM "🟢 Package for ${version} is ready."
+        }
+        
+        packaging_push_package_to_github() {
+            if [ "$MAJOR_COMMIT_REF_NAME" != "main" ]; then
+                echo "skip pushing to github: $MAJOR_COMMIT_REF_NAME"
+                return 0
+            fi
+        
+            customer=$(release_determin_customer)
+            version=${CI_COMMIT_TAG:-$(packaging_gitlab_get_version_for_pipeline_id ${customer})}
+            release=$(echo ${version} | sed sI-I~Ig)
+        
+            cd ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/
+            asset_name="tine-$(date '+%Y.%m.%d')-$(git rev-parse --short HEAD)-nightly"
+        
+            curl "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/${version}/tine20-allinone_${release}.tar.bz2" -o "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/tine20-allinone_${release}.tar.bz2"
+        
+            release_json=$(github_create_release "$GITHUB_RELEASE_REPO_OWNER" "$GITHUB_RELEASE_REPO" "$version" "$GITHUB_RELEASE_USER" "$GITHUB_RELEASE_TOKEN")
+            if [ "$?" != "0" ]; then
+                echo "$release_json"
+                return 1
+            fi
+        
+            echo "$release"
+        
+            github_release_add_asset "$release_json" "$asset_name" "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/tine20-allinone_${release}.tar.bz2" "$GITHUB_RELEASE_USER" "$GITHUB_RELEASE_TOKEN"
+        
+            matrix_send_message $MATRIX_ROOM "🟢 Packages for ${version} have been released to github."
+        
+            if [ "${MAJOR_COMMIT_REF_NAME}" == "2023.11" ]; then
+                matrix_send_message "!gGPNgDOyMWwSPjFFXa:matrix.org" 'We just released the new version "${CODENAME}" ${version} 🎉\nCheck https://www.tine-groupware.de/ and https://github.com/tine-groupware/tine/releases for more information and the downloads.\nYou can also pull the image from dockerhub: https://hub.docker.com/r/tinegroupware/tine'
+            fi
+        }
+        
+        packaging_push_to_vpackages() {
+            customer=$(release_determin_customer)
+            version=${CI_COMMIT_TAG:-$(packaging_gitlab_get_version_for_pipeline_id ${customer})}
+            release=$(echo ${version} | sed sI-I~Ig)
+        
+            echo "publishing ${release} (${version}) for ${customer} from ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/${version}/all.tar"
+        
+            if ! ssh ${VPACKAGES_SSH_URL} -o StrictHostKeyChecking=no -C  "sudo -u www-data curl ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/${version}/all.tar -o /tmp/${release}-source-${customer}.tar"; then
+                echo "Failed to download packages to vpackages"
+                return 1
+            fi
+        
+            if ! ssh ${VPACKAGES_SSH_URL} -o StrictHostKeyChecking=no -C  "sudo -u www-data /srv/packages.tine20.com/www/scripts/importTine20Repo.sh /tmp/${release}-source-${customer}.tar; sudo -u www-data rm -f /tmp/${release}-source-${customer}.tar"; then
+                echo "Failed to import package to repo"
+                return 1
+            fi
+        }
+        
+        packaging() {
+            CI_COMMIT_REF_NAME_ESCAPED=$(echo ${CI_COMMIT_REF_NAME} | sed sI/I-Ig)
+            version=${CI_COMMIT_TAG:-"nightly-${CI_COMMIT_REF_NAME_ESCAPED}-$(git describe --tags)"}
+            release=${version}
+        
+            if ! release_determin_customer; then
+                echo "No packages are build for major_commit_ref: $MAJOR_COMMIT_REF_NAME for version: $version"
+                return 1
+            fi
+        
+            echo "building packages ..."
+            if ! packaging_build_packages $version $release; then
+                echo "Failed to build packages."
+                return 1
+            fi
+        
+            if ! packaging_extract_all_package_tar; then
+                echo "Failed to extract tar archive."
+                return 1
+            fi
+        
+            echo "pushing packages to gitlab ..."
+            if ! packaging_push_packages_to_gitlab $version $release; then
+                echo "Failed to push to gitlab."
+                return 1
+            fi
+        
+            echo "setting ci pipeline id link"
+            if ! packaging_gitlab_set_ci_id_link $version; then
+                echo "Failed to set ci pipeline id link."
+                return 1
+            fi
+        }
+        
+        
+        
+
+
+        # lib/scripts/phpstan.sh
+        phpstan_analyse() {
+            if test "${CI_PROJECT_NAME}" == "tine20"; then
+                dir=tine20
+            else
+                dir=tine20/vendor/$(cat ${CI_PROJECT_DIR}/composer.json | jq -r '.name')/lib;
+            fi
+        
+            log "fixing symlinks ..."
+            # fix: phpstan fails if custom apps are symlinked. They need to be analysed in the vendor dir.
+            #    exclude symlinks
+            find $TINE20ROOT/tine20 -maxdepth 1 -type l -exec echo "        - '{}'" \; >> excludes;
+            #    unexclude vendor/metaways
+            find $TINE20ROOT/tine20/vendor -mindepth 1 -maxdepth 1 -type d -exec echo "        - '{}'" \; >> excludes;
+            sed -i '/tine20\/vendor\*/r excludes' $TINE20ROOT/phpstan.neon;
+            sed -i '/tine20\/vendor\/metaways/d' $TINE20ROOT/phpstan.neon;
+            rm excludes
+        
+            log "setting max processes ..."
+            sed -i "s/maximumNumberOfProcesses: 32/maximumNumberOfProcesses: $KUBERNETES_CPU_REQUEST/g" $TINE20ROOT/phpstan.neon
+        
+            log "setting up cache ..."
+            # CI_CUSTOM_CACHE_DIR is a volume shared betwean runners
+            # todo: monitor if composer.lock file hash prodoces to mutch dead cache
+            phpstan_cache_key=$(echo $MAJOR_COMMIT_REF_NAME $PHP_VERSION $(sha256sum $TINE20ROOT/tine20/composer.lock | cut -d ' ' -f 1) | sha256sum | cut -d ' ' -f 1)
+            export PHPSTAN_CACHE_DIR=${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAME}/phpstan-cache/v2/$phpstan_cache_key
+            echo PHPSTAN_CACHE_DIR=$PHPSTAN_CACHE_DIR
+            mkdir -p ${PHPSTAN_CACHE_DIR}
+            sed -i "s%tmpDir:%tmpDir: $PHPSTAN_CACHE_DIR%g" $TINE20ROOT/phpstan.neon
+            # create marker for cache cleanup
+            date --utc +%FT%TZ > $PHPSTAN_CACHE_DIR-lastused
+        
+            $TINE20ROOT/tine20/vendor/bin/phpstan --version
+            log "analyse target: $dir"
+            set -o pipefail
+            php -d memory_limit=2G $TINE20ROOT/tine20/vendor/bin/phpstan analyse --autoload-file=$TINE20ROOT/tine20/vendor/autoload.php --error-format=gitlab --no-progress -vvv $dir | tee ${CI_PROJECT_DIR}/code-quality-report.json
+        }
+
+        # lib/scripts/pipeline_status.sh
+        pipeline_status_send () {
+            errors=$(curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/test_report" | jq -c '.test_suites[] | select((.error_count>0) or .failed_count>0)| {name,error_count,failed_count}')
+        
+            ref=$(curl --header "JOB-TOKEN: $CI_JOB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/pipelines/$CI_PIPELINE_ID/" | jq -r .ref)
+        
+            message="pipeline $CI_PIPELINE_NAME #$CI_PIPELINE_ID for $ref failed with:"
+        
+            IFS=$'\n' 
+            for error in $errors; do
+                n=$(echo $error | jq -r '.name')
+                e=$(echo $error | jq -r '.error_count')
+                f=$(echo $error | jq -r '.failed_count')
+                message="$message"'\n'"+ $n  --  errors: $e failures: $f"
+            done
+        
+            message="$message"'\n'"$CI_PIPELINE_URL"
+        
+            echo "$message"
+        
+            matrix_send_message $MATRIX_ROOM "$message"
+        }
+
+        # lib/scripts/release.sh
+        release_tag() {
+            branch="$(echo "$CI_COMMIT_REF_NAME" | sed sI/I-Ig)"
+            tag_prefix="$branch-$(date '+%Y.%m.%d.')"
+        
+            last_counter="$(curl -H "Authorization: Bearer $GITLAB_TOKEN" "$CI_API_V4_URL/projects/$CI_PROJECT_ID/repository/tags?search=^$tag_prefix" | jq -r '.[].name' | sort --version-sort | tail -n 1 | awk -F '.' '{print $NF}')"
+            counter="$((${last_counter:-0}+1))"
+        
+            tag="$tag_prefix$counter"
+        
+            echo "tag: $tag"
+        
+            curl -H "Authorization: Bearer $GITLAB_TOKEN" -XPOST "$CI_API_V4_URL/projects/$CI_PROJECT_ID/repository/tags?tag_name=$tag&ref=$CI_COMMIT_SHA&message=version+$tag"
+        }
+        
+        release_tag_main_if_needed() {
+            if [ "$RELEASE_CE_TO_GITHUB" != "true" ]; then
+                echo "'RELEASE_CE_TO_GITHUB=$RELEASE_CE_TO_GITHUB' => do not tag main."
+                return
+            fi
+        
+            last_release_tag=$(github_get_latest_release_tag_name "$GITHUB_RELEASE_REPO_OWNER" "$GITHUB_RELEASE_REPO")
+            if [ $? != 0 ]; then
+                return 1
+            fi
+        
+            git fetch origin main || return 1
+        
+            commit_diff_count=$(git rev-list "$last_release_tag..origin/main" --count)
+            if [ $? != 0 ]; then
+                return 1
+            fi
+        
+            echo "origin/main and $last_release_tag differ in $commit_diff_count commits"
+        
+            if [ $commit_diff_count = 0 ]; then
+                echo "No difference, no new tag is created."
+                return 0
+            fi
+        
+            tag="$(date '+%Y.%m.%d.')$commit_diff_count"
+            echo "tagging origin/main as $tag"
+        
+            if ! git tag $tag; then
+                if [ "$(git rev-parse "$tag")" != "$(git rev-parse origin/main)" ]; then
+                    echo "tag $tag already exits, but it is ponting to a different commit."
+                    return 1
+                fi
+        
+                echo "Tag $tag already exits, for this commit. Using it..."
+            fi
+        
+            # "tag push" triggers tag pipeline which publishes the release
+            git push origin $tag || return 1
+            git push github $tag
+        }
+        
+        release_to_gitlab() {
+            tag="${CI_COMMIT_TAG}"
+            customer="$(release_determin_customer)"
+            previous_tag="$(git describe --abbrev=0 --tags HEAD~1 2> /dev/null || git fetch --unshallow --quiet && git describe --abbrev=0 --tags HEAD~1)" # if describe fails unshallow repo and try again
+        
+            release-cli create --description "$(repo_release_notes "$tag" "$previous_tag")" --tag-name "$tag" --ref "$tag" --name "$tag" \
+                --assets-link "{\"name\":\"all.tar\",\"url\":\"${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/${customer}/${tag}/all.tar\"}"
+        }
+        
+        # possible values tine20.org tine20.com <customer> ""
+        release_determin_customer () {
+            if test -z "${BASE_MAJOR_COMMIT_REF_NAME}"; then
+                # For branches without BASE_MAJOR_COMMIT_REF_NAME and CUSTOMER_MAJOR_COMMIT_REF_NAME variables
+                # todo remove is these kind of branches do not exist any more
+                if ! echo "${branch}" | grep -q '/'; then
+                    echo tine20.com
+                    return
+                else
+                    if [ $(echo "${branch}" | awk -F"/" '{print NF-1}') != 1 ]; then
+                        return 1
+                    fi
+        
+                    echo "${branch}" | cut -d '/' -f1
+                    return
+                fi
+            else
+                # For branches with BASE_MAJOR_COMMIT_REF_NAME and CUSTOMER_MAJOR_COMMIT_REF_NAME variables
+                if test -z "${CUSTOMER_MAJOR_COMMIT_REF_NAME}"; then
+                    if echo ${CI_COMMIT_TAG} | grep -Eq 'weekly'; then
+                        echo "tine20.org"
+                        return
+                    fi
+        
+                    echo "tine20.com"
+                    return
+                else
+                    echo "${CUSTOMER_MAJOR_COMMIT_REF_NAME}" | sed 's:/*$::'
+                    return
+                fi 
+            fi
+        }
+
+        # lib/scripts/repo.sh
+        repo_release_notes() {
+            tag=$1
+            previous_tag=$2
+        
+            echo '# Releasenotes'
+            echo '# Changelog'
+            ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/scripts/git/changelog.sh "$tag" "$previous_tag"
+        }
+
+        # lib/scripts/test.sh
+        test_prepare_working_dir() {
+            if [ "${TINE20ROOT}" != "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20" ]; then
+                log "test_preapre_wirking_dir is only requires the tine root to be: \${CI_BUILDS_DIR}/\${CI_PROJECT_NAMESPACE}/tine20"
+                # This function is only intended to work with the source from gitlab...
+                # intended: for the main repo => do basicly nothing. Or for customapps => clone main repo and include customapp
+                # and setup vars as if we where running in the main repo. 
+                return 1
+            fi
+        
+            if [ "${CI_PROJECT_NAME}" != "tine20" ] && [ "$CI_IS_CUSTOMAPP" != "true" ]; then
+                log "project name needs to be tine20 (for this, and) other test scrips to work"
+                # In many places ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20 is used. If we use TINE20ROOT root any where
+                # the project name can be anything. As long as TINE20ROOT points to the correct CI_POOJECT_DIR.
+                # todo remove this if. if we removed all occurents of ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20
+                return 1
+            fi
+        
+            # Clone tine repo, if pipeline runs for a customapp
+            if [ "${CI_IS_CUSTOMAPP}" == "true" ]; then
+                log "cloning tine ..."
+                git clone -b $TINE20_BRANCH --single-branch --depth 1 $TINE20_REPO_URL ${TINE20ROOT};
+            fi
+        
+            # todo: move into customapp case, and let gitlab hanled submodule init for the main repo
+            log "init git submodules ..."
+            cd ${TINE20ROOT}
+            git submodule init
+            git submodule update
+            
+            
+            # Install source customapp, if pipeline runs for a customapp
+            if [ "${CI_IS_CUSTOMAPP}" == "true" ]; then
+                # COMPOSER custom cache
+                # CI_CUSTOM_CACHE_DIR is a volume shared betwean runners
+                export COMPOSER_CACHE_DIR=${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}/composer-cache/v1/
+                mkdir -p ${COMPOSER_CACHE_DIR}
+        
+                log "instaling custom app ..."
+                customappname=$(cat ${CI_PROJECT_DIR}/composer.json | jq -r '.name')
+                pushd ${TINE20ROOT}/tine20
+        
+                composer config "repositories.ci" git "${CI_REPOSITORY_URL}";
+                composer require "$customappname dev-master#${CI_COMMIT_SHA}";
+                popd
+            fi
+        
+            # the shell should be left in the new working dir
+            cd ${TINE20ROOT}
+        }
+        
+        test_prepare_global_configs() {
+            log "prepareing global configs ..."
+            rm /etc/supervisor.d/worker.ini || true
+            rm /etc/crontabs/tine20 || true
+            /usr/sbin/confd -onetime -backend env
+            cp ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/ci/dockerimage/supervisor.d/webpack.ini /etc/supervisor.d/;
+            # todo move config dir to ${TINE20ROOT}/etc build test should be ablte to handle that
+            rsync -a -I --delete ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/etc/ /config;
+        }
+        
+        test_prepare_mail_db() {
+            log "prepareing databases for mail setup ..."
+            mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"SET GLOBAL wait_timeout=31536000; SET GLOBAL interactive_timeout=31536000"
+            mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"CREATE DATABASE IF NOT EXISTS dovecot"
+            mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"CREATE DATABASE IF NOT EXISTS postfix"
+            mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"CREATE USER IF NOT EXISTS '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';"
+            mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"GRANT ALL PRIVILEGES ON postfix.* TO '$MYSQL_USER'@'%'"
+            mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"GRANT ALL PRIVILEGES ON dovecot.* TO '$MYSQL_USER'@'%'"
+            mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" "dovecot" < /config/sql/dovecot_tables.sql
+            mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" "postfix" < $ARG_POSTFIX_INIT_SQL_PATH
+        }
+        
+        test_composer_install() {
+            log "trying to use cached vendor dir"
+            composer_lock_hash=$(cd ${TINE20ROOT}/tine20; sha1sum composer.json composer.lock | sha1sum | cut -d ' ' -f 1)
+            export VENDOR_CACHE_DIR=${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}/tine20/composer-cache/v1/${composer_lock_hash}
+        
+            mkdir -p $(dirname $VENDOR_CACHE_DIR)
+        
+            if [ -d $VENDOR_CACHE_DIR ] && [ ! -d ${TINE20ROOT}/tine20/vendor ]; then
+                log "found cached vendor dir using it..."
+                echo VENDOR_CACHE_DIR=$VENDOR_CACHE_DIR
+        
+                cp -r $VENDOR_CACHE_DIR ${TINE20ROOT}/tine20/vendor
+                # create marker for cache cleanup
+                date --utc +%FT%TZ > $VENDOR_CACHE_DIR-lastused
+                # do not return here, we need to run composer install, so it creates the customapps links for us
+            fi
+        
+            log "prepearing composer cache ..."
+            # CI_CUSTOM_CACHE_DIR is a volume shared betwean runners
+            export COMPOSER_CACHE_DIR=${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}/composer-cache/v1/
+            mkdir -p ${COMPOSER_CACHE_DIR}
+        
+            log "composer install ..."
+            pushd ${TINE20ROOT}/tine20
+            # trigger customapploader plugin, to create links
+            rm -rf vendor/metaways
+            COMPOSER_ALLOW_SUPERUSER=1 composer install --no-ansi --no-progress --no-suggest
+            popd
+        
+            if [ ! -d $VENDOR_CACHE_DIR ]; then
+                log "storing vendor dir as cache"
+                cp -r ${TINE20ROOT}/tine20/vendor $VENDOR_CACHE_DIR || log "storing vendor dir failed. continuing"
+            fi
+        }
+        
+        test_npm_install() {
+            additional_npm_args="$1"
+            log "trying to use cache..."
+            package_shrinkwrap_hash=$(cd ${TINE20ROOT}/tine20/Tinebase/js; echo $(sha1sum npm-shrinkwrap.json package.json)"$additional_npm_args" | sha1sum | cut -d ' ' -f 1)
+            export NODE_MODULE_CACHE_DIR=${CI_CUSTOM_CACHE_DIR}/${CI_PROJECT_NAMESPACE}/tine20/npm-cache/v1/${package_shrinkwrap_hash}
+        
+            mkdir -p $(dirname $NODE_MODULE_CACHE_DIR)
+        
+            if [ -d $NODE_MODULE_CACHE_DIR ] && [ ! -d ${TINE20ROOT}/tine20/Tinebase/js/node_modules ]; then
+                log "found cached node_modules using it..."
+                echo NODE_MODULE_CACHE_DIR=$NODE_MODULE_CACHE_DIR
+        
+                cp -r $NODE_MODULE_CACHE_DIR ${TINE20ROOT}/tine20/Tinebase/js/node_modules
+        
+                # create marker for cache cleanup
+                date --utc +%FT%TZ > $NODE_MODULE_CACHE_DIR-lastused
+                return 0
+            fi
+        
+            log "installing npm ..."
+            pushd ${TINE20ROOT}/tine20/Tinebase/js
+            npm --no-optional install $additional_npm_args
+            popd
+        
+            if [ ! -d $NODE_MODULE_CACHE_DIR ]; then
+                log "storing node_modles dir as cache"
+                cp -r ${TINE20ROOT}/tine20/Tinebase/js/node_modules $NODE_MODULE_CACHE_DIR || log "storing node_modles dir failed. continuing"
+            fi
+        }
+        
+        
+        test_phpunit() {
+            log "prepareing test .."
+            if [ -f ${TINE20ROOT}/scripts/postInstallGitlab.sh ]; then
+                ${TINE20ROOT}/scripts/postInstallGitlab.sh
+            fi
+            
+            php -v
+            echo ${NODE_TOTAL} ${NODE_INDEX};
+            echo cd ${TINE20ROOT}/${ARG_TEST_PATH_FROM_TINE20ROOT}
+        
+            cd ${TINE20ROOT}/${ARG_TEST_PATH_FROM_TINE20ROOT}
+        
+        
+            log "testing ..."
+            cmd="php ${TINE20ROOT}/tine20/vendor/bin/phpunit --color --log-junit ${CI_PROJECT_DIR}/phpunit-report.xml --debug";
+        
+            if test -n "${ARG_FILTER}"; then
+                cmd="${cmd} --filter ${ARG_FILTER}"
+            fi
+            
+            if test -n "${ARG_EXCLUDE_GROUP}"; then
+                cmd="${cmd} --exclude-group ${ARG_EXCLUDE_GROUP}"
+            fi
+        
+            if test -n "${ARG_GROUP}"; then
+                cmd="${cmd} --group ${ARG_GROUP}"
+            fi
+        
+            cmd="${cmd} ${ARG_TEST}";
+        
+            echo ${cmd};
+            ${cmd}
+        }
+        
+        # log in blue
+        log() {
+            echo -e "\033[0;34m"$@"\033[0m"
+        }
+
         echo -n 'wait for signal_files_ready ...'; while [ ! -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_files_ready ]; do sleep 1; done; echo ' done'
         
         cp -r /usr/share/tine20/Tinebase/js/node_modules $TINE20ROOT/Tinebase/js/node_modules
@@ -1131,5 +2182,6 @@
         touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_tine_ready
         
         # start tine
-        supervisord --nodaemon
+        supervisord --nodaemon        # lib/entry_points/tine_as_a_service.sh
+
 
diff --git a/ci/gitlab-ci/lib/entry_points/tine_as_a_service.sh b/ci/gitlab-ci/lib/entry_points/tine_as_a_service.sh
index bb8e3c95d3..9c3077dacd 100644
--- a/ci/gitlab-ci/lib/entry_points/tine_as_a_service.sh
+++ b/ci/gitlab-ci/lib/entry_points/tine_as_a_service.sh
@@ -1,4 +1,3 @@
-#!/bin/bash
 echo -n 'wait for signal_files_ready ...'; while [ ! -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_files_ready ]; do sleep 1; done; echo ' done'
 
 cp -r /usr/share/tine20/Tinebase/js/node_modules $TINE20ROOT/Tinebase/js/node_modules
diff --git a/ci/gitlab-ci/lib/scripts/matrix.sh b/ci/gitlab-ci/lib/scripts/matrix.sh
index c548aeeb7e..e972881970 100644
--- a/ci/gitlab-ci/lib/scripts/matrix.sh
+++ b/ci/gitlab-ci/lib/scripts/matrix.sh
@@ -1,4 +1,3 @@
-#!/bin/sh
 matrix_send_message() {
     roomid=$1
     message=$2
diff --git a/ci/gitlab-ci/makefile b/ci/gitlab-ci/makefile
index 120f215909..127aacee3a 100644
--- a/ci/gitlab-ci/makefile
+++ b/ci/gitlab-ci/makefile
@@ -14,12 +14,18 @@ lib.yml: $(scripts_sources) $(entry_points_sources) makefile
 		printf "\n\n" >> lib.yml; \
 	done
 	printf "  entry_points:\n" >> lib.yml;
-	for f in $(entry_points_sources); do  \
-		printf "    $$(basename $$f .sh):\n" >> lib.yml; \
-		printf "      - bash\n" >> lib.yml; \
+	for e in $(entry_points_sources); do  \
+		printf "    # $$e\n" >> lib.yml; \
+		printf "    $$(basename $$e .sh):\n" >> lib.yml; \
+		printf "      - $$(grep '^#!' $$e | cut -c3-)\n" >> lib.yml; \
 		printf "      - -c\n" >> lib.yml; \
 		printf "      - |\n" >> lib.yml; \
-		printf "        # $$f\n" >> lib.yml; \
-		sed -e 's/^/        /' $$f >> lib.yml; \
+		for f in $(scripts_sources); do  \
+			printf "        # $$f\n" >> lib.yml; \
+			sed -e 's/^/        /' $$f >> lib.yml; \
+			printf "\n\n" >> lib.yml; \
+		done; \
+		sed -e 's/^/        /' $$e >> lib.yml; \
+		printf "        # $$e\n" >> lib.yml; \
 		printf "\n\n" >> lib.yml; \
 	done
\ No newline at end of file

From 62a93d13af7ddf1eb6885c09fd4c9c557f242625 Mon Sep 17 00:00:00 2001
From: Milan Mertens <m.mertens@metaways.de>
Date: Tue, 27 Feb 2024 15:39:40 +0100
Subject: [PATCH 3/9] ci(e2etest) use test image

---
 ci/gitlab-ci/lib.yml                          | 42 ++++++-------------
 .../lib/entry_points/tine_as_a_service.sh     | 36 ++++------------
 ci/gitlab-ci/lib/scripts/test.sh              |  2 +-
 ci/gitlab-ci/test_js_jobs.yml                 | 20 ++-------
 4 files changed, 26 insertions(+), 74 deletions(-)

diff --git a/ci/gitlab-ci/lib.yml b/ci/gitlab-ci/lib.yml
index 1135012f72..ceab5ed068 100644
--- a/ci/gitlab-ci/lib.yml
+++ b/ci/gitlab-ci/lib.yml
@@ -883,7 +883,7 @@
       # lib/scripts/test.sh
       test_prepare_working_dir() {
           if [ "${TINE20ROOT}" != "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20" ]; then
-              log "test_preapre_wirking_dir is only requires the tine root to be: \${CI_BUILDS_DIR}/\${CI_PROJECT_NAMESPACE}/tine20"
+              log "test_preapre_working_dir requires the tine root to be: \${CI_BUILDS_DIR}/\${CI_PROJECT_NAMESPACE}/tine20"
               # This function is only intended to work with the source from gitlab...
               # intended: for the main repo => do basicly nothing. Or for customapps => clone main repo and include customapp
               # and setup vars as if we where running in the main repo. 
@@ -1061,7 +1061,7 @@
   entry_points:
     # lib/entry_points/tine_as_a_service.sh
     tine_as_a_service:
-      - 
+      - /bin/bash
       - -c
       - |
         # lib/scripts/cache_cleanup.sh
@@ -1942,7 +1942,7 @@
         # lib/scripts/test.sh
         test_prepare_working_dir() {
             if [ "${TINE20ROOT}" != "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20" ]; then
-                log "test_preapre_wirking_dir is only requires the tine root to be: \${CI_BUILDS_DIR}/\${CI_PROJECT_NAMESPACE}/tine20"
+                log "test_preapre_working_dir requires the tine root to be: \${CI_BUILDS_DIR}/\${CI_PROJECT_NAMESPACE}/tine20"
                 # This function is only intended to work with the source from gitlab...
                 # intended: for the main repo => do basicly nothing. Or for customapps => clone main repo and include customapp
                 # and setup vars as if we where running in the main repo. 
@@ -2117,29 +2117,20 @@
             echo -e "\033[0;34m"$@"\033[0m"
         }
 
-        echo -n 'wait for signal_files_ready ...'; while [ ! -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_files_ready ]; do sleep 1; done; echo ' done'
+        #!/bin/bash
+        echo -n 'wait for signal_mount_ready ...'; while [ ! -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_mount_ready ]; do sleep 1; done; echo ' done'
         
-        cp -r /usr/share/tine20/Tinebase/js/node_modules $TINE20ROOT/Tinebase/js/node_modules
-        cp -r /usr/share/tine20/vendor $TINE20ROOT/tine20/vendor
-        
-        # delete potentially old code, to make sure it can not be used 
-        if test "${TINE20ROOT}" != "/usr/share/tine20"; then rm -rf /usr/share/tine20; fi
-        
-        touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_node_modules_copied
-        
-        # install php deps
-        cd $TINE20ROOT/tine20
-        composer install --no-ansi --no-progress --no-suggest --no-scripts --ignore-platform-reqs
-        $TINE20ROOT/ci/scripts/install_custom_app.sh
+        test_prepare_working_dir
+        touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_files_ready
         
+        test_composer_install
         touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_php_deps_installed
         
-        # install js deps
-        npm --prefix $TINE20ROOT/tine20/Tinebase/js/ install;
+        test_npm_install
         touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_js_deps_installed
         
-        # setup configs
-        /usr/sbin/confd -onetime -backend env;
+        rm /etc/confd/conf.d/worker.inc.php.toml || true # todo: is it needed? can it be moved to test_prepare_config
+        test_prepare_global_configs
         
         # setup database
         if ! tine20_await_db; then
@@ -2148,20 +2139,11 @@
             exit 1
         fi
         
-        mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"CREATE DATABASE IF NOT EXISTS dovecot";
-        mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"CREATE DATABASE IF NOT EXISTS postfix";
-        mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"CREATE USER IF NOT EXISTS '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';";
-        mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"GRANT ALL PRIVILEGES ON postfix.* TO '$MYSQL_USER'@'%'";
-        mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"GRANT ALL PRIVILEGES ON dovecot.* TO '$MYSQL_USER'@'%'";
-        mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" "dovecot" < /config/sql/dovecot_tables.sql;
-        mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" "postfix" < /config/sql/postfix_tables.sql;
+        test_prepare_mail_db
         
         # setup tine enviroment
         touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAMESPACE}/tine20.log
         chown tine20:tine20 ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAMESPACE}/tine20.log
-        rm /etc/supervisor.d/worker.ini || true # todo delete when merged with "speared node container" change
-        rm /etc/crontabs/tine20 || true
-        rm /etc/confd/conf.d/worker.inc.php.toml || true 
         
         echo -n 'wait for signal_js_deps_installed ...'; while [ ! -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_js_deps_installed ]; do sleep 1; done; echo ' done'
         
diff --git a/ci/gitlab-ci/lib/entry_points/tine_as_a_service.sh b/ci/gitlab-ci/lib/entry_points/tine_as_a_service.sh
index 9c3077dacd..c09328017c 100644
--- a/ci/gitlab-ci/lib/entry_points/tine_as_a_service.sh
+++ b/ci/gitlab-ci/lib/entry_points/tine_as_a_service.sh
@@ -1,26 +1,17 @@
-echo -n 'wait for signal_files_ready ...'; while [ ! -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_files_ready ]; do sleep 1; done; echo ' done'
+#!/bin/bash
+echo -n 'wait for signal_mount_ready ...'; while [ ! -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_mount_ready ]; do sleep 1; done; echo ' done'
 
-cp -r /usr/share/tine20/Tinebase/js/node_modules $TINE20ROOT/Tinebase/js/node_modules
-cp -r /usr/share/tine20/vendor $TINE20ROOT/tine20/vendor
-
-# delete potentially old code, to make sure it can not be used 
-if test "${TINE20ROOT}" != "/usr/share/tine20"; then rm -rf /usr/share/tine20; fi
-
-touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_node_modules_copied
-
-# install php deps
-cd $TINE20ROOT/tine20
-composer install --no-ansi --no-progress --no-suggest --no-scripts --ignore-platform-reqs
-$TINE20ROOT/ci/scripts/install_custom_app.sh
+test_prepare_working_dir
+touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_files_ready
 
+test_composer_install
 touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_php_deps_installed
 
-# install js deps
-npm --prefix $TINE20ROOT/tine20/Tinebase/js/ install;
+test_npm_install
 touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_js_deps_installed
 
-# setup configs
-/usr/sbin/confd -onetime -backend env;
+rm /etc/confd/conf.d/worker.inc.php.toml || true # todo: is it needed? can it be moved to test_prepare_config
+test_prepare_global_configs
 
 # setup database
 if ! tine20_await_db; then
@@ -29,20 +20,11 @@ if ! tine20_await_db; then
     exit 1
 fi
 
-mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"CREATE DATABASE IF NOT EXISTS dovecot";
-mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"CREATE DATABASE IF NOT EXISTS postfix";
-mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"CREATE USER IF NOT EXISTS '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD';";
-mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"GRANT ALL PRIVILEGES ON postfix.* TO '$MYSQL_USER'@'%'";
-mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" -e"GRANT ALL PRIVILEGES ON dovecot.* TO '$MYSQL_USER'@'%'";
-mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" "dovecot" < /config/sql/dovecot_tables.sql;
-mysql -h$MAIL_DB_HOST -uroot -p"$MYSQL_ROOT_PASSWORD" "postfix" < /config/sql/postfix_tables.sql;
+test_prepare_mail_db
 
 # setup tine enviroment
 touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAMESPACE}/tine20.log
 chown tine20:tine20 ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/${CI_PROJECT_NAMESPACE}/tine20.log
-rm /etc/supervisor.d/worker.ini || true # todo delete when merged with "speared node container" change
-rm /etc/crontabs/tine20 || true
-rm /etc/confd/conf.d/worker.inc.php.toml || true 
 
 echo -n 'wait for signal_js_deps_installed ...'; while [ ! -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_js_deps_installed ]; do sleep 1; done; echo ' done'
 
diff --git a/ci/gitlab-ci/lib/scripts/test.sh b/ci/gitlab-ci/lib/scripts/test.sh
index 95c7caff55..2ee30fe173 100644
--- a/ci/gitlab-ci/lib/scripts/test.sh
+++ b/ci/gitlab-ci/lib/scripts/test.sh
@@ -1,6 +1,6 @@
 test_prepare_working_dir() {
     if [ "${TINE20ROOT}" != "${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20" ]; then
-        log "test_preapre_wirking_dir is only requires the tine root to be: \${CI_BUILDS_DIR}/\${CI_PROJECT_NAMESPACE}/tine20"
+        log "test_preapre_working_dir requires the tine root to be: \${CI_BUILDS_DIR}/\${CI_PROJECT_NAMESPACE}/tine20"
         # This function is only intended to work with the source from gitlab...
         # intended: for the main repo => do basicly nothing. Or for customapps => clone main repo and include customapp
         # and setup vars as if we where running in the main repo. 
diff --git a/ci/gitlab-ci/test_js_jobs.yml b/ci/gitlab-ci/test_js_jobs.yml
index 3f27a0960d..7674c28827 100644
--- a/ci/gitlab-ci/test_js_jobs.yml
+++ b/ci/gitlab-ci/test_js_jobs.yml
@@ -1,8 +1,7 @@
 .js-e2e-tests-source:
   extends: .abstract_jobs.test
   stage: test
-  needs:
-    - docker_build_source
+  needs: []
   variables:
     KUBERNETES_CPU_REQUEST: $ABSTRACT_JOBS_E2E_TEST_KUBERNETES_CPU_REQUEST
     KUBERNETES_CPU_LIMIT: $ABSTRACT_JOBS_E2E_TEST_KUBERNETES_CPU_LIMIT
@@ -19,25 +18,21 @@
     TEST_DOCKER: "true"
     TEST_BROWSER_LANGUAGE: en
     TINE20_INSTALL_LANG: de
-    ARG_COPY_SOURCE: "true"
-    ARG_IMAGE: test-source
     TINE20ROOT: ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20
     WAIT_FOR_DB_TIMEOUT: '120'
   services:
     - !reference [.lib-services, db_service]
     - !reference [.lib-services, redis_service]
     - !reference [.lib-services, mail_service]
-    - name: ${REGISTRY}/${ARG_IMAGE}-commit:${IMAGE_TAG}
+    - name: ${CI_REGISTRY}/${CI_PROJECT_NAMESPACE}/tine20/test:${BASE_MAJOR_COMMIT_REF_NAME}-${PHP_VERSION}
       alias: tine
       entrypoint:
         !reference [.lib, entry_points, tine_as_a_service]
   image:
     name: dockerregistry.metaways.net/tine20/tine20/puppeteer:1.0.4
   before_script:
-    # clone tine repo if test is run for a custom app. the tine repo will in any case be under $TINE20ROOT
-    - if test "${CI_PROJECT_NAME}" != "tine20"; then git clone -b $TINE20_BRANCH --single-branch --depth 1 $TINE20_REPO_URL $TINE20ROOT; fi
-    - cd $TINE20ROOT; git submodule init; git submodule update
-    - touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_files_ready
+    - touch ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_mount_ready
+    - echo -n 'wait for signal_files_ready ...'; while [ ! -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_files_ready ]; do sleep 1; done; echo ' done'
     # install e2e test dependencies
     - npm --prefix $TINE20ROOT/tests/e2etests/ install
     # wait for service to become ready
@@ -46,7 +41,6 @@
     - if [ -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_wait_for_database_failed ]; then echo "waiting for database failed"; exit 1; fi
     - if [ -f ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/signal_demo_data_install_failed ]; then echo "demo data install failed"; exit 1; fi
   script:
-    - env
     - |
       if [ "${TINE20_BUILDTYPE}" == "DEVELOPMENT" ]; then
         export TEST_URL=http://tine:10443;
@@ -111,12 +105,6 @@ js-e2e-tests-source-save-screenshots:
     - !reference [.js-e2e-tests-source, rules]
 js-e2e-tests-source-php-version-matrix:
   extends: .js-e2e-tests-source
-  variables:
-    IMAGE_TAG: ${CI_PIPELINE_ID}-${PHP_VERSION}
-  needs:
-    - 'docker_build_source_matrix: [7.4]'
-    - 'docker_build_source_matrix: [8.0]'
-    - 'docker_build_source_matrix: [8.1]'
   parallel:
     matrix:
       - PHP_VERSION:

From bf7ab13ec3cc52cd150b9982f8588c92dea5342e Mon Sep 17 00:00:00 2001
From: Milan Mertens <m.mertens@metaways.de>
Date: Thu, 7 Mar 2024 12:18:32 +0100
Subject: [PATCH 4/9] ci(customapps): stop building source image

---
 ci/gitlab-ci/abstract_customapp.yml | 39 +----------------------------
 1 file changed, 1 insertion(+), 38 deletions(-)

diff --git a/ci/gitlab-ci/abstract_customapp.yml b/ci/gitlab-ci/abstract_customapp.yml
index 2bfcd952fe..b2c736fb70 100644
--- a/ci/gitlab-ci/abstract_customapp.yml
+++ b/ci/gitlab-ci/abstract_customapp.yml
@@ -13,8 +13,6 @@ stages:
   - merge
 
 variables:
-  # also needs to be set in build job
-  IMAGE_TAG: $CI_PROJECT_ID-$CI_PIPELINE_IID-$PHP_VERSION
   CI_IS_CUSTOMAPP: "true"
   # TESTS
   PHP_UNIT_ALL_TESTS_SOURCE_PARALLEL: "false"
@@ -39,39 +37,4 @@ workflow:
     # Do not run pipelines on push for feature branches. Only merge requests should run feature branche pipelines.
     - if: $CI_PIPELINE_SOURCE == "push" && $CI_COMMIT_REF_NAME =~ /(^feat)|(^pu)|(change)/
       when: never
-    - when: always
-
-docker_build_source:
-  stage: build
-  trigger:
-    project: tine20/tine20
-    branch: $TINE20_BRANCH
-    strategy: depend
-  variables:
-    RUN_NO_TESTS: "true"
-    AUTO_MERGE: "never"
-    IMAGE_TAG: $CI_PROJECT_ID-$CI_PIPELINE_IID-$PHP_VERSION
-    DOCKER_BUILD_SOURCE: "true"
-docker_build_source_matrix:
-  stage: build
-  parallel:
-    matrix:
-      - PHP_VERSION:
-          - "7.4"
-          - "8.0"
-          - "8.1"
-  script:
-    - echo job is there for needs
-  rules:
-    - if: $NOT_TURE == "true"
-      when: on_success
-    - when: never
-
-docker_build_built:
-  stage: build
-  script:
-    - echo job is there for needs
-  rules:
-    - if: $NOT_TURE == "true"
-      when: on_success
-    - when: never
\ No newline at end of file
+    - when: always
\ No newline at end of file

From 0c10e95eb35b1c11038f40852fdb6b4bc75a1a68 Mon Sep 17 00:00:00 2001
From: Milan Mertens <m.mertens@metaways.de>
Date: Thu, 7 Mar 2024 12:19:27 +0100
Subject: [PATCH 5/9] ci(build): build base image during dev and built image
 build

---
 ci/gitlab-ci/build_jobs.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/ci/gitlab-ci/build_jobs.yml b/ci/gitlab-ci/build_jobs.yml
index 1b37f5ece7..a97d7352d4 100644
--- a/ci/gitlab-ci/build_jobs.yml
+++ b/ci/gitlab-ci/build_jobs.yml
@@ -236,10 +236,15 @@ docker_build_source_arm64:
 docker_build_built:
   extends: .abstract_jobs.docker
   stage: build2
+  needs: []
   script:
     - !reference [.lib, script]
+    - docker_build_image base $(docker_image_hash base)
     - export BASE_IMAGE="${REGISTRY}/base:$(docker_image_hash base)";
+    - docker_build_image dependency $(docker_image_hash dependency)
     - export DEPENDENCY_IMAGE="${REGISTRY}/dependency:$(docker_image_hash dependency)";
+    - cp $DOCKER_GIT_CONFIG ./ci/dockerimage/.gitconfig
+    - ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/ci/scripts/reuse_or_build_image.sh source false
     - ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/ci/scripts/build_image.sh build
     - ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/ci/scripts/build_image.sh built
     - ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/ci/scripts/build_image.sh test-built
@@ -307,6 +312,7 @@ docker_build_dev:
   stage: build2
   script:
     - !reference [.lib, script]
+    - docker_build_image base $(docker_image_hash base)
     - export BASE_IMAGE="${REGISTRY}/base:$(docker_image_hash base)";
     - ${CI_BUILDS_DIR}/${CI_PROJECT_NAMESPACE}/tine20/ci/scripts/build_image.sh dev
   rules:

From d012e261ab5ed69a4ef71232b932f3f7c6399fc8 Mon Sep 17 00:00:00 2001
From: Milan Mertens <m.mertens@metaways.de>
Date: Thu, 7 Mar 2024 12:20:39 +0100
Subject: [PATCH 6/9] ci(merge): stop triggering merge with docker build

---
 ci/gitlab-ci/lib.yml              | 2 --
 ci/gitlab-ci/lib/scripts/merge.sh | 1 -
 2 files changed, 3 deletions(-)

diff --git a/ci/gitlab-ci/lib.yml b/ci/gitlab-ci/lib.yml
index ceab5ed068..5d33bdbbea 100644
--- a/ci/gitlab-ci/lib.yml
+++ b/ci/gitlab-ci/lib.yml
@@ -499,7 +499,6 @@
               curl -X POST -F token=$MERGE_TRIGGER_TOKEN \
                   -F ref=$ref \
                   -F "variables[$var]=true" \
-                  -F "variables[DOCKER_BUILD_SOURCE]=true" \
                   -F "variables[SEND_PIPELINE_STATUS]=true" \
                   "$CI_API_V4_URL/projects/$CI_PROJECT_ID/trigger/pipeline" > /dev/null
           done
@@ -1558,7 +1557,6 @@
                 curl -X POST -F token=$MERGE_TRIGGER_TOKEN \
                     -F ref=$ref \
                     -F "variables[$var]=true" \
-                    -F "variables[DOCKER_BUILD_SOURCE]=true" \
                     -F "variables[SEND_PIPELINE_STATUS]=true" \
                     "$CI_API_V4_URL/projects/$CI_PROJECT_ID/trigger/pipeline" > /dev/null
             done
diff --git a/ci/gitlab-ci/lib/scripts/merge.sh b/ci/gitlab-ci/lib/scripts/merge.sh
index c367fbd668..914f4fb1fd 100644
--- a/ci/gitlab-ci/lib/scripts/merge.sh
+++ b/ci/gitlab-ci/lib/scripts/merge.sh
@@ -55,7 +55,6 @@ merge_trigger_next () {
         curl -X POST -F token=$MERGE_TRIGGER_TOKEN \
             -F ref=$ref \
             -F "variables[$var]=true" \
-            -F "variables[DOCKER_BUILD_SOURCE]=true" \
             -F "variables[SEND_PIPELINE_STATUS]=true" \
             "$CI_API_V4_URL/projects/$CI_PROJECT_ID/trigger/pipeline" > /dev/null
     done

From fdeea8739b024798424c48496c2d5ecc0bc4358e Mon Sep 17 00:00:00 2001
From: Milan Mertens <m.mertens@metaways.de>
Date: Thu, 7 Mar 2024 12:21:45 +0100
Subject: [PATCH 7/9] ci(tests): disable php version

---
 ci/gitlab-ci/test_jobs.yml | 71 +++++++++++++++++++-------------------
 1 file changed, 36 insertions(+), 35 deletions(-)

diff --git a/ci/gitlab-ci/test_jobs.yml b/ci/gitlab-ci/test_jobs.yml
index c966a9916e..5fc42fe8ed 100644
--- a/ci/gitlab-ci/test_jobs.yml
+++ b/ci/gitlab-ci/test_jobs.yml
@@ -1,38 +1,39 @@
-check-php-version:
-  extends: .abstract_jobs.copy_source
-  stage: test
-  variables:
-    ARG_IMAGE: test-source
-  needs:
-    - docker_build_source
-  script:
-    - echo "PHP version shoud be ${PHP_VERSION}."
-    - php -r "echo 'PHP version is'.PHP_MAJOR_VERSION.'.'.PHP_MINOR_VERSION;"
-    - php -r "if (version_compare(PHP_MAJOR_VERSION.'.'.PHP_MINOR_VERSION, '${PHP_VERSION}', '!=')) exit(1);"
-  rules:
-    - if: $RUN_NO_TESTS == "true"
-      when: never
-    - if: $CHECK_PHP_VERSION == "never"
-      when: never
-    - if: $RUN_ALL_TESTS == "true"
-      when: on_success
-    - if: $CHECK_PHP_VERSION == "true"
-      when: on_success
-    - when: never
-  rules:
-    - if: $RUN_NO_TESTS == "true"
-      when: never
-    - if: $CHECK_PHP_VERSION == "false"
-      when: never
-    - if: $CHECK_PHP_VERSION == "true"
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CHECK_PHP_VERSION_MERGE_REQUEST == "true"
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /test-with-health-check/
-    - if: $PIPELINE_TYPE =~ /default-tests/
-    - if: $PIPELINE_TYPE =~ /mr-tests/
-    - if: $CI_COMMIT_TAG
-  allow_failure: true
-  timeout: 20m # time run: 4m
-  interruptible: true
+# todo: should run ob build image and on test images after they are build
+# check-php-version:
+#   extends: .abstract_jobs.copy_source
+#   stage: test
+#   variables:
+#     ARG_IMAGE: test-source
+#   needs:
+#     - docker_build_source
+#   script:
+#     - echo "PHP version shoud be ${PHP_VERSION}."
+#     - php -r "echo 'PHP version is'.PHP_MAJOR_VERSION.'.'.PHP_MINOR_VERSION;"
+#     - php -r "if (version_compare(PHP_MAJOR_VERSION.'.'.PHP_MINOR_VERSION, '${PHP_VERSION}', '!=')) exit(1);"
+#   rules:
+#     - if: $RUN_NO_TESTS == "true"
+#       when: never
+#     - if: $CHECK_PHP_VERSION == "never"
+#       when: never
+#     - if: $RUN_ALL_TESTS == "true"
+#       when: on_success
+#     - if: $CHECK_PHP_VERSION == "true"
+#       when: on_success
+#     - when: never
+#   rules:
+#     - if: $RUN_NO_TESTS == "true"
+#       when: never
+#     - if: $CHECK_PHP_VERSION == "false"
+#       when: never
+#     - if: $CHECK_PHP_VERSION == "true"
+#     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CHECK_PHP_VERSION_MERGE_REQUEST == "true"
+#     - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_LABELS =~ /test-with-health-check/
+#     - if: $PIPELINE_TYPE =~ /default-tests/
+#     - if: $PIPELINE_TYPE =~ /mr-tests/
+#     - if: $CI_COMMIT_TAG
+#   allow_failure: true
+#   timeout: 20m # time run: 4m
+#   interruptible: true
 
 phpstan-analysis:
   stage: test

From 92efccafed7a9e8b1533599fcd096025194d501e Mon Sep 17 00:00:00 2001
From: Milan Mertens <m.mertens@metaways.de>
Date: Thu, 7 Mar 2024 12:22:33 +0100
Subject: [PATCH 8/9] ci(phpunit): cleanup needs

---
 ci/gitlab-ci/test_php_jobs.yml | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/ci/gitlab-ci/test_php_jobs.yml b/ci/gitlab-ci/test_php_jobs.yml
index f442e8efbd..b3cdfb7f09 100644
--- a/ci/gitlab-ci/test_php_jobs.yml
+++ b/ci/gitlab-ci/test_php_jobs.yml
@@ -47,12 +47,6 @@ php-unit-all-tests-source-matrix:
   variables:
     NODE_TOTAL: 1
     NODE_INDEX: 1
-    IMAGE_TAG: ${CI_PIPELINE_ID}-${PHP_VERSION}
-  needs:
-    - docker_build_source
-    - 'docker_build_source_matrix: [7.4]'
-    - 'docker_build_source_matrix: [8.0]'
-    - 'docker_build_source_matrix: [8.1]'
   parallel:
     matrix:
       - DATABASE_IMAGE:
@@ -156,8 +150,6 @@ php-unit-servertests-source:
   variables:
     ARG_IMAGE: test-source
     ARG_COPY_SOURCE: "true"
-  needs:
-    - docker_build_source
   timeout: 2h
   interruptible: true
   rules:
@@ -300,7 +292,6 @@ php-unit-nogitlabci-tests-built-parallel:
       when: never
     - !reference [.php-unit-nogitlabci-tests-built, rules]
 
-
 php-unit-update-tests:
   extends: .abstract_jobs.php_unit
   variables:

From ad0071203d09e18f48355cda7bd0ab5752e86b3f Mon Sep 17 00:00:00 2001
From: Christian Feitl <c.feitl@metwaways.net>
Date: Thu, 7 Mar 2024 16:05:52 +0100
Subject: [PATCH 9/9] fix(e2etests): random fail on relationPanel

---
 tests/e2etests/src/test/Tinebase/relationGridPanel.test.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/e2etests/src/test/Tinebase/relationGridPanel.test.js b/tests/e2etests/src/test/Tinebase/relationGridPanel.test.js
index 6a104c803d..7e33198dae 100644
--- a/tests/e2etests/src/test/Tinebase/relationGridPanel.test.js
+++ b/tests/e2etests/src/test/Tinebase/relationGridPanel.test.js
@@ -12,6 +12,7 @@ describe('Mainpage', () => {
     test('openEditDialog', async () => {
         let popupWindow = await lib.getEditDialog('Kontakt hinzufügen');
         await expect(popupWindow).toClick('span', {text: new RegExp("Verknüpfungen.*")});
+        await popupWindow.waitForSelector('.x-grid3-hd.x-grid3-cell.x-grid3-td-remark');
         let arrows = await popupWindow.$$('.x-panel.x-wdgt-pickergrid.x-grid-panel .x-form-trigger.x-form-arrow-trigger');
         await arrows[0].click();
         //await popupWindow.waitForSelector('.x-layer.x-combo-list',{visible: true});