From 31cf841913df58669e89893fd410ea70c2a5b665 Mon Sep 17 00:00:00 2001 From: Le Roux Bodenstein Date: Wed, 18 Dec 2024 15:57:05 +0000 Subject: [PATCH] feat(e2e): Smoke tests task & initial entrypoint COMPASS-8531 (#6571) * initial smoketest task & entrypoint * limit to just ubuntu and just the compass distribution for now * correct package * fine. no shebang * xor the IS_* platform options * verify once * add error callback --- .evergreen/buildvariants-and-tasks.in.yml | 63 ++++ .evergreen/buildvariants-and-tasks.yml | 23 ++ .evergreen/functions.yml | 25 ++ .evergreen/preinstall.sh | 5 + packages/compass-e2e-tests/.gitignore | 1 + packages/compass-e2e-tests/package.json | 3 +- packages/compass-e2e-tests/smoke-test.ts | 343 +++++++++++++++++++ packages/compass-e2e-tests/tsconfig.json | 5 +- packages/hadron-build/commands/download.d.ts | 1 + packages/hadron-build/commands/download.js | 13 +- packages/hadron-build/commands/info.d.ts | 1 + 11 files changed, 474 insertions(+), 9 deletions(-) create mode 100755 packages/compass-e2e-tests/smoke-test.ts create mode 100644 packages/hadron-build/commands/download.d.ts create mode 100644 packages/hadron-build/commands/info.d.ts diff --git a/.evergreen/buildvariants-and-tasks.in.yml b/.evergreen/buildvariants-and-tasks.in.yml index 44c5f8195e6..b9370851955 100644 --- a/.evergreen/buildvariants-and-tasks.in.yml +++ b/.evergreen/buildvariants-and-tasks.in.yml @@ -60,6 +60,40 @@ const PACKAGE_BUILD_VARIANTS = [ } ]; +const SMOKETEST_BUILD_VARIANTS = [ + { + name: 'smoketest-ubuntu', + display_name: 'Smoketest Ubuntu', + run_on: 'ubuntu2004-large', + depends_on: 'package-ubuntu', + }, + +// { +// name: 'smoketest-windows', +// display_name: 'Smoketest Windows', +// run_on: 'windows-vsCurrent-large', +// depends_on: 'package-windows', +// }, +// { +// name: 'smoketest-rhel', +// display_name: 'Smoketest RHEL', +// run_on: 'rhel80-large', +// depends_on: 'package-rhel', +// }, +// { +// name: 'smoketest-macos-x64', +// display_name: 'Smoketest MacOS Intel', +// run_on: 'macos-14', +// depends_on: 'package-macos-x64', +// }, +// { +// name: 'smoketest-macos-arm', +// display_name: 'Smoketest MacOS Arm64', +// run_on: 'macos-14-arm64', +// depends_on: 'package-macos-arm', +// } +]; + const TEST_PACKAGED_APP_BUILD_VARIANTS = [ { name: 'test-packaged-app-ubuntu', @@ -180,6 +214,19 @@ buildvariants: <% } %> <% } %> +<% for (const buildVariant of SMOKETEST_BUILD_VARIANTS) { %> +<% for (const distribution of ['compass']) { %> + - name: <%= buildVariant.name %>-<%= distribution %> + display_name: <%= buildVariant.display_name %> (<%= distribution %>) + run_on: <%= buildVariant.run_on %> + depends_on: + - name: package-<%= distribution %> + variant: <%= buildVariant.depends_on %> + tasks: + - name: smoketest-<%= distribution %> +<% } %> +<% } %> + - name: test-eol-servers display_name: Test EoL Servers run_on: ubuntu1804-large @@ -424,6 +471,22 @@ tasks: - func: save-all-artifacts vars: compass_distribution: <%= distribution %> + <% } %> + +<% for (const distribution of ['compass']) { %> + - name: smoketest-<%= distribution %> + tags: ['required-for-publish', 'run-on-pr'] + commands: + - func: prepare + - func: install + - func: bootstrap + vars: + scope: 'compass-e2e-tests' + - func: smoketest-packaged-app + vars: + mongodb_version: latest-enterprise + compass_distribution: <%= distribution %> + debug: 'compass-e2e-tests*,electron*,hadron*,mongo*' <% } %> <% for (const serverVersion of SERVER_VERSIONS) { %> diff --git a/.evergreen/buildvariants-and-tasks.yml b/.evergreen/buildvariants-and-tasks.yml index 708ebac375a..b5987af6c34 100644 --- a/.evergreen/buildvariants-and-tasks.yml +++ b/.evergreen/buildvariants-and-tasks.yml @@ -76,6 +76,14 @@ buildvariants: - name: package-compass - name: package-compass-isolated - name: package-compass-readonly + - name: smoketest-ubuntu-compass + display_name: Smoketest Ubuntu (compass) + run_on: ubuntu2004-large + depends_on: + - name: package-compass + variant: package-ubuntu + tasks: + - name: smoketest-compass - name: test-eol-servers display_name: Test EoL Servers run_on: ubuntu1804-large @@ -472,6 +480,21 @@ tasks: - func: save-all-artifacts vars: compass_distribution: compass-readonly + - name: smoketest-compass + tags: + - required-for-publish + - run-on-pr + commands: + - func: prepare + - func: install + - func: bootstrap + vars: + scope: compass-e2e-tests + - func: smoketest-packaged-app + vars: + mongodb_version: latest-enterprise + compass_distribution: compass + debug: compass-e2e-tests*,electron*,hadron*,mongo* - name: test-server-40x-community-1 tags: - required-for-publish diff --git a/.evergreen/functions.yml b/.evergreen/functions.yml index f0925ebba01..245309cfac5 100644 --- a/.evergreen/functions.yml +++ b/.evergreen/functions.yml @@ -651,6 +651,31 @@ functions: npm run --unsafe-perm --workspace compass-e2e-tests test-packaged-ci + smoketest-packaged-app: + - command: shell.exec + # Fail the task if it's idle for 10 mins + timeout_secs: 600 + params: + working_dir: src + shell: bash + env: + <<: *compass-env + DEBUG: ${debug|} + MONGODB_VERSION: ${mongodb_version|} + MONGODB_RUNNER_VERSION: ${mongodb_version|} + HADRON_DISTRIBUTION: ${compass_distribution} + script: | + set -e + # Load environment variables + eval $(.evergreen/print-compass-env.sh) + + if [[ "$IS_OSX" == "true" ]]; then + echo "Disabling clipboard usage in e2e tests (TODO: https://jira.mongodb.org/browse/BUILD-14780)" + export COMPASS_E2E_DISABLE_CLIPBOARD_USAGE="true" + fi + + npm run --workspace compass-e2e-tests smoketest + test-web-sandbox: - command: shell.exec # Fail the task if it's idle for 10 mins diff --git a/.evergreen/preinstall.sh b/.evergreen/preinstall.sh index 3a8a0382227..3093b9063c0 100755 --- a/.evergreen/preinstall.sh +++ b/.evergreen/preinstall.sh @@ -12,6 +12,11 @@ echo "NPM_VERSION: $NPM_VERSION" echo "APPDATA: $APPDATA" echo "PATH: $PATH" +# these are super useful if you want to run the smoke tests locally +echo "DEV_VERSION_IDENTIFIER: $DEV_VERSION_IDENTIFIER" +echo "EVERGREEN_BUCKET_NAME: $EVERGREEN_BUCKET_NAME" +echo "EVERGREEN_BUCKET_KEY_PREFIX: $EVERGREEN_BUCKET_KEY_PREFIX" + echo "IS_OSX: $IS_OSX" echo "IS_LINUX: $IS_LINUX" echo "IS_WINDOWS: $IS_WINDOWS" diff --git a/packages/compass-e2e-tests/.gitignore b/packages/compass-e2e-tests/.gitignore index 74d7b19ca26..ae91dfc1056 100644 --- a/packages/compass-e2e-tests/.gitignore +++ b/packages/compass-e2e-tests/.gitignore @@ -2,3 +2,4 @@ .log fixtures/*.csv fixtures/*.json +hadron-build-info.json diff --git a/packages/compass-e2e-tests/package.json b/packages/compass-e2e-tests/package.json index 94d1722ffcd..d58961e2fb8 100644 --- a/packages/compass-e2e-tests/package.json +++ b/packages/compass-e2e-tests/package.json @@ -28,7 +28,8 @@ "test-web": "env DEBUG=hadron*,mongo*,compass*,xvfb-maybe* npm run test web", "test-web-noserver": "env DEBUG=hadron*,mongo*,compass*,xvfb-maybe* npm run test web -- --disable-start-stop --bail", "coverage-merge": "nyc merge .log/coverage .nyc_output/coverage.json", - "coverage-report": "npm run coverage-merge && nyc report" + "coverage-report": "npm run coverage-merge && nyc report", + "smoketest": "ts-node smoke-test.ts" }, "devDependencies": { "@electron/rebuild": "^3.7.1", diff --git a/packages/compass-e2e-tests/smoke-test.ts b/packages/compass-e2e-tests/smoke-test.ts new file mode 100755 index 00000000000..67a9cd2de64 --- /dev/null +++ b/packages/compass-e2e-tests/smoke-test.ts @@ -0,0 +1,343 @@ +#!/usr/bin/env npx ts-node +import { createWriteStream, existsSync, promises as fs } from 'fs'; +import path from 'path'; +import yargs from 'yargs'; +import type { Argv } from 'yargs'; +import { hideBin } from 'yargs/helpers'; +import https from 'https'; +import { pick } from 'lodash'; +import { handler as writeBuildInfo } from 'hadron-build/commands/info'; + +const argv = yargs(hideBin(process.argv)) + .scriptName('smoke-tests') + .detectLocale(false) + .version(false) + .strict() + .option('bucketName', { + type: 'string', + default: () => process.env.EVERGREEN_BUCKET_NAME, + }) + .option('bucketKeyPrefix', { + type: 'string', + default: () => process.env.EVERGREEN_BUCKET_KEY_PREFIX, + }) + .option('devVersion', { + type: 'string', + // For dev versions we need this from evergreen. For beta or stable (or by + // default, ie. when testing a locally packaged app) we get it from the + // package.json + default: () => process.env.DEV_VERSION_IDENTIFIER, + }) + .option('isWindows', { + type: 'boolean', + default: () => process.env.IS_WINDOWS === 'true', + }) + .option('isOSX', { + type: 'boolean', + default: () => process.env.IS_OSX === 'true', + }) + .option('isRHEL', { + type: 'boolean', + default: () => process.env.IS_RHEL === 'true', + }) + .option('isUbuntu', { + type: 'boolean', + default: () => process.env.IS_UBUNTU === 'true', + }) + .option('arch', { + type: 'string', + choices: ['x64', 'arm64'], + demandOption: true, + default: () => process.env.ARCH ?? process.arch, + }) + .option('skipDownload', { + type: 'boolean', + description: "Don't download all assets before starting", + }) + .option('extension', { + type: 'string', + description: + 'If specified it will only run the smoke tests for the specified installer/package', + }) + .check((argv) => { + if (!argv.skipDownload) { + if (!(argv.bucketName && argv.bucketKeyPrefix)) { + throw new Error( + 'Either supply EVERGREEN_BUCKET_NAME and EVERGREEN_BUCKET_KEY_PREFIX or specify --skip-download' + ); + } + } + + // hadron-build info can only do one platform & arch at a time + const platformsCount = [ + argv.isWindows, + argv.isOSX, + argv.isUbuntu, + argv.isRHEL, + ].filter((x) => x).length; + if (platformsCount !== 1) { + throw new Error( + 'Set exactly one of IS_WINDOWS, IS_OSX, IS_UBUNTU, IS_RHEL to true' + ); + } + + return true; + }); + +type BuilderCallbackParsedArgs Argv> = + ReturnType['parseSync']>; + +type SmokeTestsContext = BuilderCallbackParsedArgs; + +async function run() { + const parsedArgs = argv.parse(); + + if ('then' in parsedArgs && typeof parsedArgs.then === 'function') { + throw new Error('Async args parser is not allowed'); + } + + const context = parsedArgs as SmokeTestsContext; + + console.log( + 'context', + pick(context, [ + 'skipDownload', + 'bucketName', + 'bucketKeyPrefix', + 'devVersion', + 'isWindows', + 'isOSX', + 'isRHEL', + 'isUbuntu', + 'arch', + 'extension', + ]) + ); + + const compassDir = path.resolve(__dirname, '..', '..', 'packages', 'compass'); + // use the specified DEV_VERSION_IDENTIFIER if set or load version from packages/compass/package.json + const version = context.devVersion + ? context.devVersion + : (JSON.parse( + await fs.readFile(path.join(compassDir, 'package.json'), 'utf8') + ).version as string); + const platform = platformFromContext(context); + const outPath = path.resolve(__dirname, 'hadron-build-info.json'); + + // build-info + const infoArgs = { + format: 'json', + dir: compassDir, + version, + platform, + arch: context.arch, + out: outPath, + }; + console.log('infoArgs', infoArgs); + writeBuildInfo(infoArgs); + const buildInfo = JSON.parse(await fs.readFile(infoArgs.out, 'utf8')); + + // filter the extensions given the platform (isWindows, isOSX, isUbuntu, isRHEL) and extension + const { isWindows, isOSX, isRHEL, isUbuntu, extension } = context; + + const packages = getFilteredPackages(compassDir, buildInfo, { + isWindows, + isOSX, + isRHEL, + isUbuntu, + extension, + }); + + if (!context.skipDownload) { + await Promise.all( + packages.map(async ({ name, filepath }) => { + await fs.mkdir(path.dirname(filepath), { recursive: true }); + const url = `https://${context.bucketName}.s3.amazonaws.com/${context.bucketKeyPrefix}/${name}`; + console.log(url); + return downloadFile(url, filepath); + }) + ); + } + + verifyPackagesExist(packages); + + // TODO(COMPASS-8533): extract or install each package and then test the Compass binary +} + +function platformFromContext( + context: SmokeTestsContext +): typeof process.platform { + if (context.isWindows) { + return 'win32'; + } + + if (context.isOSX) { + return 'darwin'; + } + + if (context.isRHEL || context.isUbuntu) { + return 'linux'; + } + + return process.platform; +} + +type PackageFilterConfig = Pick< + SmokeTestsContext, + 'isWindows' | 'isOSX' | 'isRHEL' | 'isUbuntu' | 'extension' +>; + +// subsets of the hadron-build info result + +const windowsFilenameKeys = [ + 'windows_setup_filename', + 'windows_msi_filename', + 'windows_zip_filename', + 'windows_nupkg_full_filename', +] as const; +type WindowsBuildInfo = Record; + +const osxFilenameKeys = ['osx_dmg_filename', 'osx_zip_filename'] as const; +type OSXBuildInfo = Record; + +const ubuntuFilenameKeys = [ + 'linux_deb_filename', + 'linux_tar_filename', +] as const; +type UbuntuBuildInfo = Record; + +const rhelFilenameKeys = ['linux_rpm_filename', 'rhel_tar_filename'] as const; +type RHELBuildInfo = Record; + +function buildInfoIsWindows(buildInfo: any): buildInfo is WindowsBuildInfo { + for (const key of windowsFilenameKeys) { + if (!buildInfo[key]) { + return false; + } + } + return true; +} + +function buildInfoIsOSX(buildInfo: any): buildInfo is OSXBuildInfo { + for (const key of osxFilenameKeys) { + if (!buildInfo[key]) { + return false; + } + } + return true; +} + +function buildInfoIsUbuntu(buildInfo: any): buildInfo is UbuntuBuildInfo { + for (const key of ubuntuFilenameKeys) { + if (!buildInfo[key]) { + return false; + } + } + return true; +} + +function buildInfoIsRHEL(buildInfo: any): buildInfo is RHELBuildInfo { + for (const key of rhelFilenameKeys) { + if (!buildInfo[key]) { + return false; + } + } + return true; +} + +type Package = { + name: string; + filepath: string; +}; + +function getFilteredPackages( + compassDir: string, + buildInfo: any, + config: PackageFilterConfig +): Package[] { + let names: string[] = []; + + if (config.isWindows) { + if (!buildInfoIsWindows(buildInfo)) { + throw new Error('missing windows package keys'); + } + names = windowsFilenameKeys.map((key) => buildInfo[key]); + } else if (config.isOSX) { + if (!buildInfoIsOSX(buildInfo)) { + throw new Error('missing osx package keys'); + } + names = osxFilenameKeys.map((key) => buildInfo[key]); + } else if (config.isRHEL) { + if (!buildInfoIsRHEL(buildInfo)) { + throw new Error('missing rhel package keys'); + } + names = rhelFilenameKeys.map((key) => buildInfo[key]); + } else if (config.isUbuntu) { + if (!buildInfoIsUbuntu(buildInfo)) { + throw new Error('missing ubuntu package keys'); + } + names = ubuntuFilenameKeys.map((key) => buildInfo[key]); + } + + const extension = config.extension; + + return names + .filter((name) => !extension || name.endsWith(extension)) + .map((name) => { + return { + name, + filepath: path.join(compassDir, 'dist', name), + }; + }); +} + +async function downloadFile(url: string, targetFile: string): Promise { + return await new Promise((resolve, reject) => { + https + .get(url, (response) => { + const code = response.statusCode ?? 0; + + if (code >= 400) { + return reject(new Error(response.statusMessage)); + } + + // handle redirects + if (code > 300 && code < 400 && !!response.headers.location) { + return resolve(downloadFile(response.headers.location, targetFile)); + } + + // save the file to disk + const fileWriter = createWriteStream(targetFile) + .on('finish', () => { + resolve(); + }) + .on('error', (error: any) => { + reject(error); + }); + + response.pipe(fileWriter); + }) + .on('error', (error: any) => { + reject(error); + }); + }); +} + +function verifyPackagesExist(packages: Package[]): void { + for (const { filepath } of packages) { + if (!existsSync(filepath)) { + throw new Error( + `${filepath} does not exist. Did you forget to download or package?` + ); + } + } +} + +run() + .then(function () { + console.log('done'); + }) + .catch(function (err) { + console.error(err.stack); + process.exit(1); + }); diff --git a/packages/compass-e2e-tests/tsconfig.json b/packages/compass-e2e-tests/tsconfig.json index d8494d7522d..8534724e776 100644 --- a/packages/compass-e2e-tests/tsconfig.json +++ b/packages/compass-e2e-tests/tsconfig.json @@ -1,7 +1,4 @@ { "extends": "@mongodb-js/tsconfig-compass/tsconfig.common.json", - "compilerOptions": { - // https://webdriver.io/docs/typescript/#framework-setup - //"types": ["node", "@wdio/globals/types"] - } + "compilerOptions": {} } diff --git a/packages/hadron-build/commands/download.d.ts b/packages/hadron-build/commands/download.d.ts new file mode 100644 index 00000000000..110182f0c73 --- /dev/null +++ b/packages/hadron-build/commands/download.d.ts @@ -0,0 +1 @@ +export function run(argv: any): Promise; diff --git a/packages/hadron-build/commands/download.js b/packages/hadron-build/commands/download.js index fd0468f4657..4632957d7b9 100644 --- a/packages/hadron-build/commands/download.js +++ b/packages/hadron-build/commands/download.js @@ -23,8 +23,7 @@ const builder = { }, }; -const handler = function handler(argv) { - cli.argv = argv; +const run = async function run (argv) { argv.version = argv.version.replace(/^v/, ''); const assets = Target.getAssetsForVersion(argv.dir, argv.version); @@ -43,12 +42,18 @@ const handler = function handler(argv) { cli.info(`${asset.name}: download from evg bucket complete`); }); - Promise.all(downloads).catch(abortIfError); -}; + return Promise.all(downloads) +} + +const handler = function handler(argv) { + cli.argv = argv; + run(argv).catch(abortIfError); +} module.exports = { command, describe, builder, + run, handler, }; diff --git a/packages/hadron-build/commands/info.d.ts b/packages/hadron-build/commands/info.d.ts new file mode 100644 index 00000000000..e0d57f58fc9 --- /dev/null +++ b/packages/hadron-build/commands/info.d.ts @@ -0,0 +1 @@ +export function handler(argv: any): void;