CLIENT-3171 CLIENT-3192 CLIENT-3202 CLIENT-3206 Support multi-record …

…transactions (#140)

.github/actions/run-ee-server/action.yml

@@ -0,0 +1,77 @@
+name: 'Run EE Server'
+description: 'Run EE server. Returns once server is ready. Only tested on Linux and macOS'
+# NOTE: do not share this server container with others
+# since it's using the default admin / admin credentials
+inputs:
+  # All inputs in composite actions are strings
+  use-server-rc:
+    required: true
+    description: Deploy server release candidate?
+    default: 'false'
+  server-tag:
+    required: true
+    description: Specify Docker tag
+    default: 'latest'
+  # Github Composite Actions can't access secrets
+  # so we need to pass them in as inputs
+  docker-hub-username:
+    description: Required for using release candidates
+    required: false
+  docker-hub-password:
+    description: Required for using release candidates
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+  - name: Log into Docker Hub to get server RC
+    if: ${{ inputs.use-server-rc == 'true' }}
+    run: docker login --username ${{ inputs.docker-hub-username }} --password ${{ inputs.docker-hub-password }}
+    shell: bash
+
+  - run: echo IMAGE_NAME=aerospike/aerospike-server-enterprise${{ inputs.use-server-rc == 'true' && '-rc' || '' }}:${{ inputs.server-tag }} >> $GITHUB_ENV
+    shell: bash
+
+  - run: echo NEW_IMAGE_NAME=${{ env.IMAGE_NAME }}-security-and-sc >> $GITHUB_ENV
+    shell: bash
+
+  # macOS Github runners and Windows self-hosted runners don't have buildx installed by default
+  - if: ${{ runner.os == 'Windows' || runner.os == 'macOS' }}
+    uses: docker/setup-buildx-action@v3
+
+  - name: Build and push
+    uses: docker/build-push-action@v6
+    with:
+      # Don't want to use default Git context or else it will clone the whole client repo again
+      context: .github/workflows/docker-build-context
+      build-args: |
+        server_image=${{ env.IMAGE_NAME }}
+      tags: ${{ env.NEW_IMAGE_NAME }}
+      # setup-buildx-action configures Docker to use the docker-container build driver
+      # This driver doesn't publish an image locally by default
+      # so we have to manually enable it
+      load: true
+
+  - run: echo SERVER_CONTAINER_NAME="aerospike" >> $GITHUB_ENV
+    shell: bash
+
+  - run: docker run -d --name ${{ env.SERVER_CONTAINER_NAME }} -e DEFAULT_TTL=2592000 -p 3000:3000 ${{ env.NEW_IMAGE_NAME }}
+    shell: bash
+
+  - uses: ./.github/actions/wait-for-as-server-to-start
+    with:
+      container-name: ${{ env.SERVER_CONTAINER_NAME }}
+      is-security-enabled: false
+      is-strong-consistency-enabled: true
+
+  # All the partitions are assumed to be dead when reusing a roster file
+  - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm --enable --execute "manage revive ns test"
+    shell: bash
+
+  # Apply changes
+  - run: docker exec ${{ env.SERVER_CONTAINER_NAME }} asadm --enable --execute "manage recluster"
+    shell: bash
+
+  # For debugging
+  - run: docker logs aerospike
+    shell: bash

.github/actions/wait-for-as-server-to-start/action.yml

@@ -0,0 +1,28 @@
+name: 'Wait for Aerospike server to start'
+description: Only tested on Linux and macOS
+inputs:
+  container-name:
+    required: true
+  is-security-enabled:
+    required: false
+    default: 'false'
+  is-strong-consistency-enabled:
+    required: false
+    default: 'false'
+
+runs:
+  using: "composite"
+  steps:
+  - name: 'macOS: install timeout command'
+    if: ${{ runner.os == 'macOS' }}
+    run: brew install coreutils
+    shell: bash
+
+  # Composite actions doesn't support step-level timeout-minutes
+  # Use timeout command and store polling logic in file to make it easier to read
+  # Call bash shell explicitly since timeout uses "sh" shell by default, for some reason
+  # Also, we don't want to fail if we timeout in case the server *did* finish starting up but the script couldn't detect it due to a bug
+  # Effectively, this composite action is like calling "sleep" that is optimized to exit early when it detects an ok from the server
+  - name: Wait for EE server to start
+    run: timeout 30 bash ./.github/workflows/wait-for-as-server-to-start.bash ${{ inputs.container-name }} ${{ inputs.is-security-enabled }} ${{ inputs.is-strong-consistency-enabled }} || true
+    shell: bash

.github/workflows/build-artifacts.yml

@@ -0,0 +1,70 @@
+name: Build artifacts
+run-name: Build artifacts (run_tests=${{ inputs.run_tests }}, use-server-rc=${{ inputs.use-server-rc }}, server-tag=${{ inputs.server-tag }})
+
+on:
+  workflow_dispatch:
+    inputs:
+      run_tests:
+        description: "Run integration tests?"
+        required: true
+        type: boolean
+        default: false
+      use-server-rc:
+        type: boolean
+        required: true
+        default: false
+        description: 'Test against server release candidate? (e.g to test new server features)'
+      server-tag:
+        type: string
+        required: true
+        default: 'latest'
+        description: 'Server docker image tag (e.g to test a client backport version)'
+
+  workflow_call:
+    inputs:
+      # The "dev" tests test the artifacts against a server
+      run_tests:
+        required: false
+        type: boolean
+        default: false
+      # workflow_call hack
+      is_workflow_call:
+        type: boolean
+        default: true
+        required: false
+      # This input is only used in workflow_call events
+      sha-to-build-and-test:
+        description: A calling workflow may want to run this workflow on a different ref than the calling workflow's ref
+        type: string
+        # Make it required to make things simple
+        required: true
+      # A calling workflow doesn't actually set values to the inputs below
+      # But that workflow needs to have default values for these inputs
+      use-server-rc:
+        required: false
+        default: false
+        type: boolean
+      server-tag:
+        type: string
+        required: false
+        default: 'latest'
+    secrets:
+      DOCKER_HUB_BOT_USERNAME:
+        required: true
+      DOCKER_HUB_BOT_PW:
+        required: true
+      MAC_M1_SELF_HOSTED_RUNNER_PW:
+        required: true
+
+jobs:
+  dotnet:
+    strategy:
+      fail-fast: false
+    uses: ./.github/workflows/dotnet.yml
+    with:
+      # Can't use env context here, so just copy from build-sdist env var
+      sha-to-build-and-test: ${{ inputs.is_workflow_call == true && inputs.sha-to-build-and-test || github.sha }}
+      run_tests: ${{ inputs.run_tests }}
+      use-server-rc: ${{ inputs.use-server-rc }}
+      server-tag: ${{ inputs.server-tag }}
+    secrets: inherit

.github/workflows/docker-build-context/Dockerfile

@@ -0,0 +1,39 @@
+ARG server_image=aerospike/aerospike-server-enterprise
+ARG ROSTER_FILE_NAME=roster.smd
+# Temp file for passing node id from one build stage to another
+# Docker doesn't support command substitution for setting values for ARG variables, so we have to do this
+ARG NODE_ID_FILE_NAME=node_id
+
+FROM $server_image as configure-server
+
+WORKDIR /opt/aerospike/smd
+
+# Enable authentication
+
+ARG AEROSPIKE_CONF_TEMPLATE_PATH=/etc/aerospike/aerospike.template.conf
+
+# Enable strong consistency
+RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency true/" $AEROSPIKE_CONF_TEMPLATE_PATH
+RUN sed -i "s/\(namespace.*{\)/\1\n\tstrong-consistency-allow-expunge true/" $AEROSPIKE_CONF_TEMPLATE_PATH
+ARG ROSTER_FILE_NAME
+COPY $ROSTER_FILE_NAME .
+
+# Fetch node id from roster.smd
+
+# There's no tag for the latest major version to prevent breaking changes in jq
+# This is the next best thing
+FROM ghcr.io/jqlang/jq:1.7 as get-jq
+# jq docker image doesn't have a shell
+# We need a shell to fetch and pass the node id to the next build stage
+FROM busybox as get-node-id
+COPY --from=get-jq /jq /bin/
+ARG ROSTER_FILE_NAME
+COPY $ROSTER_FILE_NAME .
+ARG NODE_ID_FILE_NAME
+RUN jq --raw-output '.[1].value' $ROSTER_FILE_NAME > $NODE_ID_FILE_NAME
+
+FROM configure-server as set-node-id
+ARG NODE_ID_FILE_NAME
+COPY --from=get-node-id $NODE_ID_FILE_NAME .
+RUN sed -i "s/\(^service {\)/\1\n\tnode-id $(cat $NODE_ID_FILE_NAME)/" $AEROSPIKE_CONF_TEMPLATE_PATH
+RUN rm $NODE_ID_FILE_NAME

.github/workflows/docker-build-context/roster.smd

@@ -0,0 +1,12 @@
+[
+   [
+      97107025374203,
+      1
+   ],
+   {
+      "key": "test",
+      "value": "a1",
+      "generation": 1,
+      "timestamp": 465602976982
+   }
+]

.github/workflows/docker-build-context/security.smd

@@ -0,0 +1,48 @@
+[
+   [
+      162276881999406,
+      14
+   ],
+   {
+      "key": "admin|P",
+      "value": "$2a$10$7EqJtq98hPqEX7fNZaFWoO1mVO/4MLpGzsqojz6E9Gef6iXDjXdDa",
+      "generation": 1,
+      "timestamp": 0
+   },
+   {
+      "key": "admin|R|user-admin",
+      "value": "",
+      "generation": 1,
+      "timestamp": 0
+   },
+   {
+      "key": "superuser|P",
+      "value": "$2a$10$7EqJtq98hPqEX7fNZaFWoOZX0o4mZCBUwvzt/iecIcG4JaDOC41zK",
+      "generation": 3,
+      "timestamp": 458774922440
+   },
+   {
+      "key": "superuser|R|read-write-udf",
+      "value": "",
+      "generation": 3,
+      "timestamp": 458774922441
+   },
+   {
+      "key": "superuser|R|sys-admin",
+      "value": "",
+      "generation": 3,
+      "timestamp": 458774922442
+   },
+   {
+      "key": "superuser|R|user-admin",
+      "value": "",
+      "generation": 3,
+      "timestamp": 458774922442
+   },
+   {
+      "key": "superuser|R|data-admin",
+      "value": null,
+      "generation": 2,
+      "timestamp": 458774718056
+   }
+]

.github/workflows/tests.yml

@@ -0,0 +1,58 @@
+name: Run tests
+
+# Trigger test workflow whenever:
+# 1. A pull request is updated (e.g with new commits)
+# 2. Commits are pushed directly to the stage or master branch
+on:
+  push:
+    branches: ["stage*", "master*"]
+  pull_request:
+    branches: ["stage*", "master*"]
+    types: [
+      # Default triggers
+      opened,
+      synchronize,
+      reopened,
+      # Additional triggers
+      labeled,
+      unlabeled
+    ]
+  workflow_dispatch:
+    inputs:
+      test-server-rc:
+        type: boolean
+        default: false
+        required: true
+
+jobs:
+
+  test-ee:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+
+    - uses: ./.github/actions/run-ee-server
+      with:
+        use-server-rc: ${{ contains(github.event.pull_request.labels.*.name, 'new-server-features') }}
+        docker-hub-username: ${{ secrets.DOCKER_HUB_BOT_USERNAME }}
+        docker-hub-password: ${{ secrets.DOCKER_HUB_BOT_PW }}
+
+    - name: Setup .NET
+      uses: actions/setup-dotnet@v4
+      with:
+        dotnet-version: 6.0.x
+
+    - name: Restore dependencies
+      run: dotnet restore /p:EnableWindowsTargeting=true
+
+    - name: Build
+      run: dotnet build --configuration Release --no-restore /p:EnableWindowsTargeting=true
+
+    - name: Run tests
+      run: dotnet test --configuration Release --no-build --verbosity normal
+
+    - name: Show logs if failed
+      if: ${{ failure() }}
+      run: |
+        docker container logs aerospike
+        cat ./configs/aerospike.conf

.github/workflows/wait-for-as-server-to-start.bash

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -x
+# Makes sure that if the "docker exec" command fails, it is not ignored
+set -o pipefail
+
+container_name=$1
+is_security_enabled=$2
+
+if [[ $is_security_enabled == true ]]; then
+    # We need to pass credentials to asinfo if server requires it
+    # TODO: passing in credentials via command line flags since I can't figure out how to use --instance with global astools.conf
+    user_credentials="--user=admin --password=admin"
+fi
+
+while true; do
+    # An unset variable will have a default empty value
+    # Intermediate step is to print docker exec command's output in case it fails
+    # Sometimes, errors only appear in stdout and not stderr, like if asinfo throws an error because of no credentials
+    # (This is a bug in asinfo since all error messages should be sent to stderr)
+    # But piping and passing stdin to grep will hide the first command's stdout.
+    # grep doesn't have a way to print all lines passed as input.
+    # ack does have an option but it doesn't come installed by default
+    # shellcheck disable=SC2086 # The flags in user credentials should be separate anyways. Not one string
+    echo "Checking if we can reach the server via the service port..."
+    if docker exec "$container_name" asinfo $user_credentials -v status | tee >(cat) | grep -qE "^ok"; then
+        # Server is ready when asinfo returns ok
+        echo "Can reach server now."
+        # docker container inspect "$container_name"
+        break
+    fi
+
+    echo "Server didn't return ok via the service port. Polling again..."
+done
+
+# Although the server may be reachable via the service port, the cluster may not be fully initialized yet.
+# If we try to connect too soon (e.g right after "status" returns ok), the client may throw error code -1
+while true; do
+    echo "Waiting for server to stabilize (i.e return a cluster key)..."
+    # We assume that when an ERROR is returned, the cluster is not stable yet (i.e not fully initialized)
+    if docker exec "$container_name" asinfo $user_credentials -v cluster-stable 2>&1 | (! grep -qE "^ERROR"); then
+        echo "Server is in a stable state."
+        break
+    fi
+
+    echo "Server did not return a cluster key. Polling again..."
+done