feat: add support for ADOT [WIP]

.github/workflows/check-new-releases.yaml

@@ -51,6 +51,46 @@ jobs:
           name: releases-${{ matrix.artifact_prefix }}.json
           path: releases.json
 
+  check-adot-releases:
+    name: Check new AWS DIstro for OpenTelemetry releases
+    strategy:
+      fail-fast: false
+    runs-on: ubuntu-latest
+    outputs:
+      releases: ${{ steps.list-releases.outputs.releases }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Install dependencies
+        env:
+          NODE_MAJOR: 20
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ca-certificates curl gnupg
+          sudo mkdir -p /etc/apt/keyrings
+          curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | sudo gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg      
+          echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_${NODE_MAJOR}.x nodistro main" | sudo tee /etc/apt/sources.list.d/nodesource.list
+          sudo apt-get update
+          sudo apt-get install nodejs -y
+      - name: Build dependency checker application
+        run: |
+          (cd .github/workflows/scripts/list-releases && npm install && npm run build)
+      - name: List matching releases
+        id: list-releases
+        env:
+          DISTRO_NAME: adot
+          GH_REPOSITORY: aws-observability/aws-otel-collector
+        run: |
+          (cd .github/workflows/scripts/list-releases && npm run start) > releases.json
+          cat releases.json
+      - name: Upload releases.json artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: releases-adot.json
+          path: releases.json
+
   open-pr:
     name: Open PR
     needs: check-community-releases

.github/workflows/scripts/list-releases/src/index.ts

@@ -3,6 +3,10 @@ import { Octokit } from '@octokit/rest';
 // Declare that this is a module
 export {};
 
+interface GitHubReleaseAsset {
+    name: string;
+}
+
 interface Release {
     version: string;
     artifact: string;
@@ -13,16 +17,14 @@ declare global {
         interface ProcessEnv {
             DISTRO_NAME: string;
             GH_REPOSITORY: string;
-            GH_ASSET_PREFIX: string;
-            GH_ASSET_SUFFIX: string;
+            GH_ASSET_PREFIX?: string;
+            GH_ASSET_SUFFIX?: string;
         }
     }
 }
 
 const distroName = process.env.DISTRO_NAME;
 const [ owner, repo ] = process.env.GH_REPOSITORY.split('/');
-const assetPrefix = process.env.GH_ASSET_PREFIX;
-const assetSuffix = process.env.GH_ASSET_SUFFIX;
 
 (async () => {
     const octokit = new Octokit();
@@ -35,11 +37,41 @@ const assetSuffix = process.env.GH_ASSET_SUFFIX;
 
     for await (const { data: releases } of iterator) {
         for (const release of releases) {
-            let matchingAsset = release.assets.find((asset) => asset.name?.startsWith(assetPrefix) && asset.name?.endsWith(assetSuffix));
+            let matchingAsset: string = '';
+
+            switch(distroName) {
+                case 'otelcol-core':
+                case 'otelcol-contrib':
+                    const assetPrefix = process.env.GH_ASSET_PREFIX;
+                    if (!assetPrefix) {
+                        throw new Error('The required "GH_ASSET_PREFIX" environment variable is not set');
+                    }
+
+                    const assetSuffix = process.env.GH_ASSET_SUFFIX;
+                    if (!assetSuffix) {
+                        throw new Error('The required "GH_ASSET_SUFFIX" environment variable is not set');
+                    }
+
+                    matchingAsset = (release.assets.find((asset : GitHubReleaseAsset) => asset.name?.startsWith(assetPrefix) && asset.name?.endsWith(assetSuffix))?.name);
+                    break;
+                case 'adot':
+                    /*
+                     * ADOT is linking RPM packages in the body of the release. e.g.:
+                     * https://aws-otel-collector.s3.amazonaws.com/amazon_linux/amd64/v0.33.2/aws-otel-collector.rpm
+                     */
+                    matchingAsset = [...release.body?.toString().matchAll(/https?:\/\/\S+/g)].find(value => {
+                        const url = value.toString();
+                        return url.includes('/amazon_linux/amd64/') && url.endsWith('rpm');
+                    })[0]
+                    break;
+                default:
+                    throw new Error(`Unknown distro name '${distroName}'`);
+            }
+
             if (matchingAsset) {
                 foundReleases.push({
                     version: release.tag_name || '',
-                    artifact: matchingAsset.name,
+                    artifact: matchingAsset,
                 });
             }
         }

validation-src/src/assets/supported-distributions.json

@@ -30,5 +30,21 @@
                 "artifact": "otelcol-contrib_0.85.0_linux_amd64.rpm"
             }
         ]
+    },
+    "adot": {
+        "provider": "Amazon Web Services (AWS)",
+        "description": "AWS Distro for OpenTelemetry",
+        "website": "https://aws.amazon.com/otel/",
+        "repository": "aws-observability/aws-otel-collector",
+        "releases": [
+            {
+                "version": "v0.33.1",
+                "artifact": "https://aws-otel-collector.s3.amazonaws.com/amazon_linux/amd64/v0.33.1/aws-otel-collector.rpm"
+            },
+            {
+                "version": "v0.32.1",
+                "artifact": "https://aws-otel-collector.s3.amazonaws.com/amazon_linux/amd64/v0.32.1/aws-otel-collector.rpm"
+            }
+        ]
     }
 }

validation-src/src/images/otelcol-validator/src/index.ts

@@ -1,4 +1,5 @@
 import { APIGatewayProxyResult, APIGatewayEvent } from 'aws-lambda';
+import { spawn } from 'node:child_process';
 import spawnAsync from '@expo/spawn-async';
 import { realpath, writeFile } from 'fs/promises';
 
@@ -10,16 +11,76 @@ interface SpawnError extends Error {
     signal: string;
 }
 
-export const validate = async (configuration: string): Promise<void> => {
-    const configPath = '/tmp/config.yaml';
+declare global {
+    namespace NodeJS {
+        interface ProcessEnv {
+            DISTRO_NAME: string;
+        }
+    }
+}
+
+const distroName = process.env.DISTRO_NAME;
+
+export const validateAdot = async (otelcolRealPath: string, configPath: string): Promise<void> => {
+    /*
+     * ADOT does not support the `validate` subcommand
+     * (see https://github.com/aws-observability/aws-otel-collector/issues/2391),
+     * so we need to fire up the collector an scan for a known log line that
+     * signifies succesful bootstrap.
+     */
+    let resolveFn: Function, rejectFn: Function;
+
+    const res = new Promise<void>((resolve, reject) => {
+        resolveFn = resolve;
+        rejectFn = reject;
+    });
+    let isResolved = false;
+
+    const otelcol = spawn(otelcolRealPath, [`--config=${configPath}`]);
+
+    let stdout = '', stderr = '';
 
-    await writeFile(configPath, configuration!, {
-        flag: 'w+',
+    otelcol.stdout.on('data', (data) => {
+        stdout += data.toString();
     });
 
-    // Resolve real path (the collector binary is likely symlinked)
-    const otelcolRealPath = await realpath('/usr/bin/otelcol');
+    otelcol.stderr.on('data', (data) => {
+        /*
+         * If the configuration is valid, the ADOT collector outputs to stderr:
+         * `Everything is ready. Begin running and processing data.`
+         */
+        stderr += data.toString();
+
+        if (stderr.includes('Everything is ready. Begin running and processing data.')) {
+            resolveFn();
+        }
+    });
 
+    otelcol.on('close', (code) => {
+        if (!isResolved) {
+            if (code === 0) {
+                resolveFn();
+            } else {
+                const err = new Error(`The '${otelcolRealPath}' exited with status '${code}'`) as SpawnError;
+                err.status = code || -1;
+                err.stdout = stdout;
+                err.stderr = stderr;
+
+                rejectFn(err);
+            }
+        }
+    });
+
+    return res
+        .finally(() => {
+            isResolved = true;
+        }).finally(() => {
+            // Kill process, we got what we needed
+            otelcol.kill();
+        });
+}
+
+export const validateOtelCol = async (otelcolRealPath: string, configPath: string): Promise<void> => {
     await spawnAsync(otelcolRealPath, ['validate', `--config=${configPath}`], {
         ignoreStdio: false,
         detached: false,
@@ -41,7 +102,22 @@ export const handler = async (event: APIGatewayEvent): Promise<APIGatewayProxyRe
     }
 
     try {
-        await validate(config);
+        const configPath = '/tmp/config.yaml';
+
+        await writeFile(configPath, config!, {
+            flag: 'w+',
+        });
+
+        // Resolve real path (the collector binary is likely symlinked)
+        const otelcolRealPath = await realpath('/usr/bin/otelcol');
+
+        switch(distroName) {
+            case 'adot':
+                await validateAdot(otelcolRealPath, config);
+                break;
+            default:
+                await validateOtelCol(otelcolRealPath, config);
+        }
 
         return {
             statusCode: 200,

validation-src/src/main.ts

@@ -8,6 +8,14 @@ import { BucketDeployment, Source } from 'aws-cdk-lib/aws-s3-deployment';
 import { Construct } from 'constructs';
 import { join } from 'path';
 
+declare global {
+  namespace NodeJS {
+      interface ProcessEnv {
+          TEST_ENVIRONMENT_NAME: string;
+      }
+  }
+}
+
 interface Distributions {
   [id: string]: Distribution;
 }
@@ -140,6 +148,7 @@ export class OTelBinValidationStack extends Stack {
           code: DockerImageCode.fromImageAsset(join(__dirname, 'images', 'otelcol-validator'), {
             platform: Platform.LINUX_AMD64,
             buildArgs: {
+              DISTRO_NAME: id,
               GH_TOKEN: props.githubToken,
               GH_REPOSITORY: distribution.repository,
               GH_RELEASE: release.version,
@@ -148,9 +157,10 @@ export class OTelBinValidationStack extends Stack {
           }),
           /*
            * The default 128 cause the OtelCol process to swap a lot, and that increased
-           * latency by a couple seconds when testing with the Otelcol Contrib v0.85.1
+           * latency by a couple seconds in cold start and normal validations when testing
+           * with the Otelcol Contrib v0.85.1.
            */
-          memorySize: 512,
+          memorySize: 1024,
           timeout: Duration.seconds(15),
         });
 
@@ -168,6 +178,8 @@ if (!process.env.GH_TOKEN) {
   throw new Error('No GitHub token provided via the "GH_TOKEN" environment variable');
 }
 
+const testEnvironmentName = process.env.TEST_ENVIRONMENT_NAME || 'dev';
+
 const env = {
   account: process.env.CDK_DEFAULT_ACCOUNT,
   region: process.env.CDK_DEFAULT_REGION,
@@ -176,6 +188,6 @@ const env = {
 
 const app = new App();
 
-new OTelBinValidationStack(app, 'otelbin-validation-dev', env);
+new OTelBinValidationStack(app, `otelbin-validation-${testEnvironmentName}`, env);
 
 app.synth();