chore: test migration backward compatibility (#5492)

## About the changes
This PR will validate that our current migrations are backward
compatible with the latest stable release of Unleash.
It will do so by starting a database, applying the latest migrations,
and then starting a docker container with the last stable unleash
release and running UI tests against it.

There's a risk that the current version of UI tests will not work with
the previous version of our UI. Because of that we copied the previous
version of cypress tests
(https://github.com/Unleash/unleash/tree/5.6/frontend/cypress) into oss
folder and removed the ones that are enterprise only. We can discuss a
better way of doing this to avoid having to maintain this folder always
in sync with the previous version of Unleash

This action will only run when there are changes in migrations or to
cypress tests.

.github/workflows/validate-migrations.yaml

@@ -0,0 +1,44 @@
+name: Test db migrations
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'src/migrations/**'
+      - '.github/workflows/validate-migrations.yaml'
+      - 'test-migrations/**'
+      - 'frontend/cypress'
+  workflow_dispatch:
+jobs:
+  test-migrations:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Use Node.js 18.x
+        uses: actions/setup-node@v3
+        with:
+            node-version: 18.x
+            cache: 'yarn'
+      - name: Start database
+        working-directory: test-migrations
+        run: docker compose up db -d --wait -t 90
+      - name: Start stable version of Unleash
+        working-directory: test-migrations
+        run: docker compose up unleash -d --wait -t 90
+      # add some data with terraform
+      - name: Apply migrations
+        env:
+            DATABASE_URL: postgres://postgres:unleash@localhost:5432/unleash
+            DATABASE_SSL: false
+        run: |
+          yarn install --frozen-lockfile --ignore-scripts
+          yarn db-migrate up
+      # run ui tests against previous version of Unleash
+      - name: Run Cypress
+        uses: cypress-io/github-action@v5
+        with:
+          working-directory: frontend
+          env: AUTH_USER=admin,AUTH_PASSWORD=unleash4all
+          config: baseUrl=http://localhost:4242
+          spec: cypress/oss/**/*.spec.ts

frontend/cypress/global.d.ts

@@ -33,6 +33,7 @@ declare namespace Cypress {
             name: string,
             shouldWait?: boolean,
             project?: string,
+            closeSplash?: boolean, // @deprecated to support old tests
         ): Chainable;
 
         // VARIANTS

frontend/cypress/oss/feature/feature.spec.ts

@@ -0,0 +1,24 @@
+///<reference path="../../global.d.ts" />
+
+describe('feature', () => {
+    const randomId = String(Math.random()).split('.')[1];
+    const featureToggleName = `unleash-e2e-${randomId}`;
+
+    before(() => {
+        cy.runBefore();
+    });
+
+    after(() => {
+        cy.deleteFeature_API(featureToggleName);
+    });
+
+    beforeEach(() => {
+        cy.login_UI();
+        cy.visit('/features');
+    });
+
+    it('can create a feature toggle', () => {
+        cy.createFeature_UI(featureToggleName, true, 'default', true);
+        cy.url().should('include', featureToggleName);
+    });
+});

frontend/cypress/support/API.ts

@@ -31,7 +31,7 @@ export const deleteFeature_API = (
     const project = projectName || 'default';
     cy.request({
         method: 'DELETE',
-        url: `${baseUrl}/api/admin/projects/${projectName}/features/${name}`,
+        url: `${baseUrl}/api/admin/projects/${project}/features/${name}`,
     });
     return cy.request({
         method: 'DELETE',

frontend/cypress/support/UI.ts

@@ -30,7 +30,7 @@ export const login_UI = (
 ): Chainable<any> => {
     return cy.session(user, () => {
         cy.visit('/');
-        cy.wait(1500);
+        cy.wait(200);
         cy.get("[data-testid='LOGIN_EMAIL_ID']").type(user);
 
         if (AUTH_PASSWORD) {
@@ -52,21 +52,26 @@ export const createFeature_UI = (
     name: string,
     shouldWait?: boolean,
     project?: string,
+    forceInteractions?: boolean,
 ): Chainable<any> => {
     const projectName = project || 'default';
-    cy.visit(`/projects/${project}`);
-    cy.get('[data-testid=NAVIGATE_TO_CREATE_FEATURE').click();
+    const uiOpts = forceInteractions ? { force: true } : undefined;
+    cy.visit(`/projects/${projectName}`);
+    cy.get('[data-testid=NAVIGATE_TO_CREATE_FEATURE').click(uiOpts);
 
     cy.intercept('POST', `/api/admin/projects/${projectName}/features`).as(
         'createFeature',
     );
 
     cy.wait(300);
 
-    cy.get("[data-testid='CF_NAME_ID'").type(name);
-    cy.get("[data-testid='CF_DESC_ID'").type('hello-world');
-    if (!shouldWait) return cy.get("[data-testid='CF_CREATE_BTN_ID']").click();
-    else cy.get("[data-testid='CF_CREATE_BTN_ID']").click();
+    cy.get("[data-testid='CF_NAME_ID'] input").type(name, uiOpts);
+    cy.get("[data-testid='CF_DESC_ID'] textarea")
+        .first()
+        .type('hello-world', uiOpts);
+    if (!shouldWait)
+        return cy.get("[data-testid='CF_CREATE_BTN_ID']").click(uiOpts);
+    else cy.get("[data-testid='CF_CREATE_BTN_ID']").click(uiOpts);
     return cy.wait('@createFeature');
 };
 
@@ -283,7 +288,8 @@ export const addVariantsToFeature_UI = (
 ) => {
     const project = projectName || 'default';
     cy.visit(`/projects/${project}/features/${featureToggleName}/variants`);
-    cy.wait(1000);
+    cy.wait(200);
+
     cy.intercept(
         'PATCH',
         `/api/admin/projects/${project}/features/${featureToggleName}/environments/development/variants`,

frontend/package.json

@@ -24,6 +24,7 @@
     "fmt:check": "biome check src",
     "ts:check": "tsc",
     "e2e": "NODE_OPTIONS=\"${NODE_OPTIONS} --no-experimental-fetch\" yarn run cypress open --config baseUrl='http://localhost:3000' --env AUTH_USER=admin,AUTH_PASSWORD=unleash4all",
+    "e2e:oss": "yarn --cwd frontend run cypress run --spec \"cypress/oss/**/*.spec.ts\" --config baseUrl='http://localhost:4242' --env AUTH_USER=admin,AUTH_PASSWORD=unleash4all",
     "e2e:heroku": "NODE_OPTIONS=\"${NODE_OPTIONS} --no-experimental-fetch\" yarn run cypress open --config baseUrl='https://unleash.herokuapp.com' --env AUTH_USER=admin,AUTH_PASSWORD=unleash4all",
     "gen:api": "NODE_OPTIONS=\"${NODE_OPTIONS} --no-experimental-fetch\" orval --config orval.config.js",
     "gen:api:demo": "NODE_OPTIONS=\"${NODE_OPTIONS} --no-experimental-fetch\" UNLEASH_OPENAPI_URL=https://app.unleash-hosted.com/demo/docs/openapi.json yarn run gen:api",

test-migrations/docker-compose.yml

@@ -0,0 +1,47 @@
+version: "3.9"
+services:
+  # The Unleash server waits for the migrations to be applied by waiting on the 
+  # migrations container to be healthy.
+  unleash:
+    image: unleashorg/unleash-server:latest # this is the latest stable release
+    pull_policy: "always"
+    ports:
+      - "4242:4242"
+    environment:
+      DATABASE_URL: "postgres://postgres:unleash@db/unleash"
+      DATABASE_SSL: "false"
+      LOG_LEVEL: "debug"
+      INIT_ADMIN_API_TOKENS: "*:*.unleash-insecure-admin-api-token"
+    depends_on:
+      db:
+        condition: service_healthy
+    healthcheck:
+      test: wget --no-verbose --tries=1 --spider http://localhost:4242/health || exit 1
+      interval: 1s
+      timeout: 1m
+      retries: 5
+      start_period: 15s
+
+  db:
+    ports:
+      - "5432:5432"
+    expose:
+      - "5432"
+    image: postgres:16
+    environment:
+      POSTGRES_DB: "unleash"
+      # trust incoming connections blindly (DON'T DO THIS IN PRODUCTION!)
+      POSTGRES_HOST_AUTH_METHOD: "trust"
+    healthcheck:
+      test:
+        [
+          "CMD",
+          "pg_isready",
+          "--username=postgres",
+          "--host=127.0.0.1",
+          "--port=5432",
+        ]
+      interval: 2s
+      timeout: 1m
+      retries: 5
+      start_period: 10s