feat: validate the configuration against actual collector binaries running on AWS

.env.example

@@ -24,3 +24,8 @@ CLERK_SECRET_KEY=
 # Whether to show or hide the attribution notice. Defaults to `false`
 # https://reactflow.dev/docs/guides/remove-attribution/
 NEXT_PUBLIC_HIDE_REACT_FLOW_ATTRIBUTION=
+
+# OTelBin can validate the configuration against real collector distributions.
+# This URL points to such a deployed validator.
+COLLECTOR_CONFIGURATION_VALIDATION_URL=
+COLLECTOR_CONFIGURATION_VALIDATION_API_KEY=

.github/workflows/ci.yaml

@@ -10,9 +10,14 @@ on:
     branches:
       - 'main'
 
+# Ensure we only have one such workflow running per branch, to avoid
+# conflicts in the test env
+concurrency:
+  group: ${{ github.ref }}
+
 jobs:
   verify:
-    name: Verify
+    name: Code tests
     runs-on: ubuntu-latest
     timeout-minutes: 20
     steps:
@@ -21,7 +26,8 @@ jobs:
         with:
           fetch-depth: 0
 
-      - uses: actions/setup-node@v3
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
         with:
           node-version-file: .nvmrc
 
@@ -36,3 +42,118 @@ jobs:
       - name: Test
         shell: bash
         run: npm run test
+
+  prep-itests:
+    name: Deploy validation backend
+    needs: ['verify']
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    outputs:
+      test_matrix: ${{ steps.prepare_test_matrix.outputs.test_matrix }}
+      validation_api_apigateway_name: ${{ steps.parse_cdk_output.outputs.api_gateway_name }}
+      validation_api_apigateway_url: ${{ steps.parse_cdk_output.outputs.api_gateway_url }}\
+      validation_api_apigateway_key_id: ${{ steps.parse_cdk_output.outputs.api_gateway_key_id }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: .nvmrc
+
+      - name: Install CDK dependencies
+        shell: bash
+        working-directory: packages/otelbin-validation
+        run: |
+          npm ci
+          npm i -g aws-cli
+
+      - name: Get test environment name
+        id: get_test_env_name
+        shell: bash
+        # When the trigger is a pull event, use the PR's branch name (`github.head_ref`), which
+        # is available only for pull trigger. Otherwise, use the branch on which the commit was pushed.
+        # (Ref names in pull requests are in the shape of `<pr_id>/merge` and those will break CDK.)
+        run: |
+          export REF_NAME="${{ github.head_ref || github.ref_name }}"
+          # Ensure test name is not long enough to break restrictions on length names in AWS (e.g., role-name length)
+          # This logic ensures test envs have max length 18. Names of length 13 to 17 will become of length 18 to ensure
+          # no accidental clashes.
+          if [[ ${#REF_NAME} -gt 12 ]]; then
+            REF_NAME="${REF_NAME:0:12}-$(echo "${REF_NAME:12}" | md5sum | awk '{print $1}' | cut -c 1-5 | tr -d '\n')"
+          fi
+          echo "test_env_name=${REF_NAME}" >> $GITHUB_OUTPUT || exit 1
+
+      - name: Deploy validation backend
+        shell: bash
+        working-directory: packages/otelbin-validation
+        env:
+          AWS_ACCESS_KEY_ID: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ACCESS_KEY_ID_PROD || secrets.AWS_ACCESS_KEY_ID_DEV }}
+          AWS_SECRET_ACCESS_KEY: ${{ github.ref == 'refs/heads/main' && secrets.AWS_SECRET_ACCESS_KEY_PROD || secrets.AWS_SECRET_ACCESS_KEY_DEV }}
+          AWS_DEFAULT_REGION: 'us-east-2' 
+          CDK_DEPLOY_ACCOUNT: ${{ github.ref == 'refs/heads/main' && '462608073829' || '622203989445' }}
+          CDK_DEPLOY_REGION: 'us-east-2'
+          GH_TOKEN: ${{ github.token }}
+          TEST_ENVIRONMENT_NAME: ${{ steps.get_test_env_name.outputs.test_env_name }}
+        run: |
+          npx projen deploy --require-approval never --outputs-file ./cdk-outputs.json
+
+      - name: Read API Gateway URL
+        id: parse_cdk_output
+        shell: bash
+        working-directory: packages/otelbin-validation
+        env:
+          TEST_ENVIRONMENT_NAME: ${{ steps.get_test_env_name.outputs.test_env_name }}
+        run: |
+          echo api_gateway_name=$(jq -r ".[\"otelbin-validation-${TEST_ENVIRONMENT_NAME}\"] | .apiname" < ./cdk-outputs.json) >> ${GITHUB_OUTPUT}
+          echo api_gateway_url=$(jq -r ".[\"otelbin-validation-${TEST_ENVIRONMENT_NAME}\"] | .apiurl" < ./cdk-outputs.json) >> ${GITHUB_OUTPUT}
+          echo api_gateway_key_id=$(jq -r ".[\"otelbin-validation-${TEST_ENVIRONMENT_NAME}\"] | .apikeyid" < ./cdk-outputs.json) >> ${GITHUB_OUTPUT}
+
+      - name: Prepare test matrix
+        id: prepare_test_matrix
+        shell: bash
+        working-directory: packages/otelbin-validation
+        run: |
+          echo test_matrix=$(jq -r '. | [to_entries[] | {distribution: .key, release: .value.releases[].version} | "\(.distribution)/\(.release)"]' < ./src/assets/supported-distributions.json) >> ${GITHUB_OUTPUT}
+
+  run-itests:
+    name: Validation tests
+    needs: ['prep-itests']
+    strategy:
+      matrix:
+        test_matrix: ${{ fromJson(needs.prep-itests.outputs.test_matrix )}}
+      fail-fast: false # We want to run all the tests, to prevent one failure from hiding another
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version-file: .nvmrc
+
+      - name: Install CDK dependencies
+        shell: bash
+        working-directory: packages/otelbin-validation
+        run: |
+          npm ci
+
+      - name: Retrieve API Key for Validation API
+        shell: bash
+        working-directory: packages/otelbin-validation
+        env:
+          AWS_ACCESS_KEY_ID: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ACCESS_KEY_ID_PROD || secrets.AWS_ACCESS_KEY_ID_DEV }}
+          AWS_SECRET_ACCESS_KEY: ${{ github.ref == 'refs/heads/main' && secrets.AWS_SECRET_ACCESS_KEY_PROD || secrets.AWS_SECRET_ACCESS_KEY_DEV }}
+          AWS_DEFAULT_REGION: 'us-east-2' 
+          API_GATEWAY_NAME: ${{ needs.prep-itests.outputs.validation_api_apigateway_name }}
+          API_GATEWAY_URL: ${{ needs.prep-itests.outputs.validation_api_apigateway_url }}
+          RELEASE_UNDER_TEST: ${{ matrix.test_matrix }}
+        run: |
+          VALIDATION_API_KEY=$(aws apigateway get-api-key --api-key ${{ needs.prep-itests.outputs.validation_api_apigateway_key_id }} --include-value | jq -r '.value') npm run test

.github/workflows/clean-up-test-env.yaml

@@ -0,0 +1,41 @@
+name: Clean up test environment
+
+# only trigger on pull request closed events
+on:
+  pull_request:
+    types: [ closed ]
+
+jobs:
+  cleanup_test_env:
+    name: Clean up test environment ${{ github.ref }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version-file: .nvmrc
+
+      - name: Install CDK dependencies
+        shell: bash
+        working-directory: packages/otelbin-validation
+        run: |
+          npm ci
+
+      - name: Delete validation backend
+        shell: bash
+        working-directory: packages/otelbin-validation
+        env:
+          AWS_ACCESS_KEY_ID: ${{ github.ref == 'refs/heads/main' && secrets.AWS_ACCESS_KEY_ID_PROD || secrets.AWS_ACCESS_KEY_ID_DEV }}
+          AWS_SECRET_ACCESS_KEY: ${{ github.ref == 'refs/heads/main' && secrets.AWS_SECRET_ACCESS_KEY_PROD || secrets.AWS_SECRET_ACCESS_KEY_DEV }}
+          AWS_DEFAULT_REGION: 'us-east-2' 
+          CDK_DEPLOY_ACCOUNT: ${{ github.ref == 'refs/heads/main' && '462608073829' || '622203989445' }}
+          CDK_DEPLOY_REGION: 'us-east-2'
+          GH_TOKEN: ${{ github.token }}
+          TEST_ENVIRONMENT_NAME: ${{ steps.get_test_env_name.outputs.test_env_name }}
+        run: |
+          npx projen destroy --force

.github/workflows/discover-new-releases.yaml

@@ -0,0 +1,162 @@
+name: Discover new OtelCol releases
+on:
+  schedule:
+    - cron: '0 2 * * *' # run at 2 AM UTC
+  workflow_dispatch: {}
+
+jobs:
+  check-community-releases:
+    name: OpenTelemetry official distribution
+    strategy:
+      fail-fast: false
+      matrix:
+        distro:
+        - name: otelcol-core
+          artifact_prefix: otelcol_
+          artifact_suffix: _linux_amd64.rpm
+        - name: otelcol-contrib
+          artifact_prefix: otelcol-contrib_
+          artifact_suffix: _linux_amd64.rpm
+    runs-on: ubuntu-latest
+    outputs:
+      releases: ${{ steps.list-releases.outputs.releases }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install dependencies
+        env:
+          NODE_MAJOR: 20
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ca-certificates curl gnupg
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg      
+          echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
+          sudo apt-get update
+          sudo apt-get install nodejs -y
+      - name: Build list generator
+        working-directory: .github/workflows/scripts/list-releases
+        run: |
+          npm install
+          npm run build
+      - name: Look up ignored releases range
+        id: lookup-ignored-releases
+        shell: bash
+        working-directory: packages/otelbin-validation/src/assets
+        run: |
+          echo "range=$(jq -r '.["${{matrix.distro.name}}"].ignoredReleases' < supported-distributions.json)" | tee -a ${GITHUB_OUTPUT}
+      - name: List matching releases
+        id: list-releases
+        shell: bash
+        env:
+          DISTRO_NAME: ${{ matrix.distro.name }}
+          GH_REPOSITORY: open-telemetry/opentelemetry-collector-releases
+          GH_ASSET_PREFIX: ${{ matrix.distro.artifact_prefix }}
+          GH_ASSET_SUFFIX: ${{ matrix.distro.artifact_suffix }}
+          IGNORED_RELEASES: ${{ steps.lookup-ignored-releases.outputs.range }}
+        working-directory: .github/workflows/scripts/list-releases
+        run: |
+          npm run start --silent | tee releases.json
+      - name: Upload releases.json artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: releases-${{ matrix.distro.name }}
+          path: .github/workflows/scripts/list-releases/releases.json
+
+  check-adot-releases:
+    name: AWS Distro for OpenTelemetry
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    outputs:
+      releases: ${{ steps.list-releases.outputs.releases }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install dependencies
+        env:
+          NODE_MAJOR: 20
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ca-certificates curl gnupg
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg      
+          echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
+          sudo apt-get update
+          sudo apt-get install nodejs -y
+      - name: Build list generator
+        working-directory: .github/workflows/scripts/list-releases
+        run: |
+          npm install
+          npm run build
+      - name: Look up ignored releases range
+        id: lookup-ignored-releases
+        shell: bash
+        working-directory: packages/otelbin-validation/src/assets
+        run: |
+          echo "range=$(jq -r '.["adot"].ignoredReleases' < supported-distributions.json)" | tee -a ${GITHUB_OUTPUT}
+      - name: List matching releases
+        id: list-releases
+        working-directory: .github/workflows/scripts/list-releases
+        env:
+          DISTRO_NAME: adot
+          GH_REPOSITORY: aws-observability/aws-otel-collector
+          IGNORED_RELEASES: ${{ steps.lookup-ignored-releases.outputs.range }}
+        run: |
+          npm run start --silent | tee releases.json
+      - name: Upload releases.json artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: releases-adot
+          path: .github/workflows/scripts/list-releases/releases.json
+
+  compose-supported-distro-list:
+    name: Compose supported distro list 
+    needs:
+    - check-community-releases
+    - check-adot-releases
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Download release artifacts
+        uses: actions/download-artifact@v3
+        with:
+          path: releases/
+      - name: Merge release data
+        run: |
+          mv packages/otelbin-validation/src/assets/supported-distributions.json packages/otelbin-validation/src/assets/supported-distributions.json.old
+          jq -rs 'reduce .[] as $item ({}; . * $item)' packages/otelbin-validation/src/assets/supported-distributions.json.old releases/*/*.json > packages/otelbin-validation/src/assets/supported-distributions.json
+      - name: Upload updated releases.json artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: supported-distributions
+          path: packages/otelbin-validation/src/assets/supported-distributions.json
+
+  create-pr:
+    name: Create Pull Request
+    needs:
+      - compose-supported-distro-list
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Download updated supported distro list
+        uses: actions/download-artifact@v3
+        with:
+          name: supported-distributions
+          path: packages/otelbin-validation/src/assets
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v5
+        with:
+          title: Update supported versions
+          commit-message: 'chore: update the version list of supported OpenTelemetry collector distros'
+          add-paths: packages/otelbin-validation/src/assets/supported-distributions.json
+          branch: update-supported-versions
+          reviewers: mmanciop,bripkens

.github/workflows/scripts/list-releases/.gitignore

@@ -0,0 +1,2 @@
+dist/
+node_modules/