From cdbb72f5f54177cf5b1e26239bd1ca47fb2a36ea Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 01:56:17 +0530 Subject: [PATCH 01/49] build images on pr open/push --- .../preview-branches-publish-images.yml | 93 +++++++++++++++++++ .../workflows/production-publish-images.yml | 85 +++++++++-------- 2 files changed, 135 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/preview-branches-publish-images.yml diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml new file mode 100644 index 0000000000..6024f6af15 --- /dev/null +++ b/.github/workflows/preview-branches-publish-images.yml @@ -0,0 +1,93 @@ +name: Build Docker Images for Preview Branches + +on: + pull_request: + types: [opened, synchronize, reopened] + +concurrency: + group: ${{ github.head_ref }}-build-images + cancel-in-progress: true + +jobs: + build_images: + runs-on: ubuntu-latest-8-cores + permissions: + contents: read + timeout-minutes: 10 + + strategy: + matrix: + service: [multiplayer, files, connection, client, api] + fail-fast: true + + steps: + - uses: actions/checkout@v4 + + - name: Generate Build Metadata + id: build-metadata + run: | + echo "BUILD_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT + echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Login to Amazon ECR Public + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: private + + - name: Define repository name + id: repo-name + run: | + echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT + + - name: Create Private ECR Repository if not exists + id: create-ecr + env: + REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }} + run: | + aws ecr create-repository --repository-name $REPO_NAME || true + REPO_INFO=$(aws ecr describe-repositories --repository-names $REPO_NAME) + ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') + echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx-${{ matrix.service }}- + ${{ runner.os }}-buildx- + + - name: Build, Tag, and Push Image to Amazon ECR Private + env: + ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} + run: | + docker buildx build \ + --cache-from=type=local,src=/tmp/.buildx-cache \ + --cache-to=type=local,dest=/tmp/.buildx-cache-new \ + --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ + --build-arg GIT_SHA="${{ github.sha }}" \ + --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ + --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \ + --label "org.opencontainers.image.revision=${{ github.sha }}" \ + --push \ + -t $ECR_URL:pr-${{ github.event.pull_request.number }} \ + -t $ECR_URL:${{ github.head_ref }} \ + -t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \ + -f quadratic-${{ matrix.service }}/Dockerfile . + + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index 5e4315b3e9..d7e79df8aa 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -3,7 +3,6 @@ name: Build and Publish Images to ECR on: push: branches: - - self-hosting-setup #remove - main concurrency: @@ -16,45 +15,45 @@ jobs: matrix: service: [multiplayer, files, connection, client, api] steps: - - uses: actions/checkout@v4 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: us-east-1 - - - name: Login to Amazon ECR Public - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - with: - registry-type: public - - - name: Define repository name - id: repo-name - run: | - echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT - - - name: Create Public ECR Repository if not exists - id: create-ecr - env: - REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }} - run: | - aws ecr-public create-repository --repository-name $REPO_NAME || true - REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME) - ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') - echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT - - - name: Read VERSION file - id: version - run: echo "VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT - - - name: Build, Tag, and Push Image to Amazon ECR Public - env: - ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} - IMAGE_TAG: ${{ steps.version.outputs.VERSION }} - run: | - docker build -t $ECR_URL:$IMAGE_TAG -t $ECR_URL:latest -f quadratic-${{ matrix.service }}/Dockerfile . - docker push $ECR_URL:$IMAGE_TAG - docker push $ECR_URL:latest \ No newline at end of file + - uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Login to Amazon ECR Public + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Define repository name + id: repo-name + run: | + echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT + + - name: Create Public ECR Repository if not exists + id: create-ecr + env: + REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }} + run: | + aws ecr-public create-repository --repository-name $REPO_NAME || true + REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME) + ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') + echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT + + - name: Read VERSION file + id: version + run: echo "VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT + + - name: Build, Tag, and Push Image to Amazon ECR Public + env: + ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} + IMAGE_TAG: ${{ steps.version.outputs.VERSION }} + run: | + docker build -t $ECR_URL:$IMAGE_TAG -t $ECR_URL:latest -f quadratic-${{ matrix.service }}/Dockerfile . + docker push $ECR_URL:$IMAGE_TAG + docker push $ECR_URL:latest From f7d69104ac96bfffbe6b72316752c88f1c0e8aa1 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 02:22:00 +0530 Subject: [PATCH 02/49] fix branch name --- .github/workflows/preview-branches-publish-images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 6024f6af15..397561019d 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -28,6 +28,7 @@ jobs: run: | echo "BUILD_TIME=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT echo "GIT_SHA_SHORT=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + echo "BRANCH_NAME=$(echo "${{ github.head_ref }}" | tr '/' '-')" >> $GITHUB_OUTPUT - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -83,7 +84,7 @@ jobs: --label "org.opencontainers.image.revision=${{ github.sha }}" \ --push \ -t $ECR_URL:pr-${{ github.event.pull_request.number }} \ - -t $ECR_URL:${{ github.head_ref }} \ + -t $ECR_URL:${{ steps.build-metadata.outputs.BRANCH_NAME }} \ -t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \ -f quadratic-${{ matrix.service }}/Dockerfile . From e80308ab6b686bc856bff55016ce36a1cd674375 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 02:30:17 +0530 Subject: [PATCH 03/49] create cache directory --- .github/workflows/preview-branches-publish-images.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 397561019d..b3495919b2 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -60,6 +60,8 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + buildkitd-flags: --debug - name: Cache Docker layers uses: actions/cache@v3 @@ -70,6 +72,9 @@ jobs: ${{ runner.os }}-buildx-${{ matrix.service }}- ${{ runner.os }}-buildx- + - name: Create cache directory if it doesn't exist + run: mkdir -p /tmp/.buildx-cache + - name: Build, Tag, and Push Image to Amazon ECR Private env: ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} From 997c4f6d40ccb3bbe281983faf34504ff339ccde Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 02:35:41 +0530 Subject: [PATCH 04/49] init cache dir --- .../workflows/preview-branches-publish-images.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index b3495919b2..4b7ac4dc86 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -53,7 +53,13 @@ jobs: env: REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }} run: | - aws ecr create-repository --repository-name $REPO_NAME || true + # Try to describe the repository first + if ! aws ecr describe-repositories --repository-names $REPO_NAME 2>/dev/null; then + # Repository doesn't exist, create it + aws ecr create-repository --repository-name $REPO_NAME + fi + + # Get the repository URI either way REPO_INFO=$(aws ecr describe-repositories --repository-names $REPO_NAME) ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT @@ -72,8 +78,10 @@ jobs: ${{ runner.os }}-buildx-${{ matrix.service }}- ${{ runner.os }}-buildx- - - name: Create cache directory if it doesn't exist - run: mkdir -p /tmp/.buildx-cache + - name: Create and initialize buildx cache + run: | + mkdir -p /tmp/.buildx-cache + echo '{"layers":{}}' > /tmp/.buildx-cache/index.json - name: Build, Tag, and Push Image to Amazon ECR Private env: From 4cd48592a421d1f8061922457cb557bb9b57e20e Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 02:40:50 +0530 Subject: [PATCH 05/49] cache init bug --- .github/workflows/preview-branches-publish-images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 4b7ac4dc86..4441108241 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -81,7 +81,8 @@ jobs: - name: Create and initialize buildx cache run: | mkdir -p /tmp/.buildx-cache - echo '{"layers":{}}' > /tmp/.buildx-cache/index.json + CACHE_TAG="${{ runner.os }}-buildx-${{ matrix.service }}" + echo "{\"layers\":{},\"manifests\":{\"$CACHE_TAG\":{\"layers\":[]}}}" > /tmp/.buildx-cache/index.json - name: Build, Tag, and Push Image to Amazon ECR Private env: From 51f633424c3f7725e83b561587d040f058009629 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 02:49:51 +0530 Subject: [PATCH 06/49] fix private ecr login --- .github/workflows/preview-branches-publish-images.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 4441108241..1aeca25800 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -37,11 +37,9 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} aws-region: ${{ secrets.AWS_REGION }} - - name: Login to Amazon ECR Public + - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - with: - registry-type: private - name: Define repository name id: repo-name From 037d36f35180b28b962478213eeb0eb736dc290b Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 03:10:47 +0530 Subject: [PATCH 07/49] use existing ecr repo --- .github/workflows/preview-branches-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 1aeca25800..d1a4b9a0c7 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -44,7 +44,7 @@ jobs: - name: Define repository name id: repo-name run: | - echo "REPO_NAME=quadratic-${{ matrix.service }}" >> $GITHUB_OUTPUT + echo "REPO_NAME=quadratic-${{ matrix.service }}-development" >> $GITHUB_OUTPUT - name: Create Private ECR Repository if not exists id: create-ecr From e297eb42a91f57d5879f6dea871f5cc9192d3c65 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 03:21:38 +0530 Subject: [PATCH 08/49] try ecr public --- .../preview-branches-publish-images.yml | 97 ++++++++++--------- 1 file changed, 53 insertions(+), 44 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index d1a4b9a0c7..6ac2415227 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -37,70 +37,79 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} aws-region: ${{ secrets.AWS_REGION }} - - name: Login to Amazon ECR + - name: Login to Amazon ECR Public id: login-ecr uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public - name: Define repository name id: repo-name run: | echo "REPO_NAME=quadratic-${{ matrix.service }}-development" >> $GITHUB_OUTPUT - - name: Create Private ECR Repository if not exists + - name: Create Public ECR Repository if not exists id: create-ecr env: REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }} run: | # Try to describe the repository first - if ! aws ecr describe-repositories --repository-names $REPO_NAME 2>/dev/null; then + if ! aws ecr-public describe-repositories --repository-names $REPO_NAME 2>/dev/null; then # Repository doesn't exist, create it - aws ecr create-repository --repository-name $REPO_NAME + aws ecr-public create-repository --repository-name $REPO_NAME fi # Get the repository URI either way - REPO_INFO=$(aws ecr describe-repositories --repository-names $REPO_NAME) + REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME) ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - buildkitd-flags: --debug - - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-${{ matrix.service }}- - ${{ runner.os }}-buildx- - - - name: Create and initialize buildx cache - run: | - mkdir -p /tmp/.buildx-cache - CACHE_TAG="${{ runner.os }}-buildx-${{ matrix.service }}" - echo "{\"layers\":{},\"manifests\":{\"$CACHE_TAG\":{\"layers\":[]}}}" > /tmp/.buildx-cache/index.json - - - name: Build, Tag, and Push Image to Amazon ECR Private + # - name: Set up Docker Buildx + # uses: docker/setup-buildx-action@v3 + # with: + # buildkitd-flags: --debug + + # - name: Cache Docker layers + # uses: actions/cache@v3 + # with: + # path: /tmp/.buildx-cache + # key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }} + # restore-keys: | + # ${{ runner.os }}-buildx-${{ matrix.service }}- + # ${{ runner.os }}-buildx- + + # - name: Create and initialize buildx cache + # run: | + # mkdir -p /tmp/.buildx-cache + # CACHE_TAG="${{ runner.os }}-buildx-${{ matrix.service }}" + # echo "{\"layers\":{},\"manifests\":{\"$CACHE_TAG\":{\"layers\":[]}}}" > /tmp/.buildx-cache/index.json + + # - name: Build, Tag, and Push Image to Amazon ECR Private + # env: + # ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} + # run: | + # docker buildx build \ + # --cache-from=type=local,src=/tmp/.buildx-cache \ + # --cache-to=type=local,dest=/tmp/.buildx-cache-new \ + # --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ + # --build-arg GIT_SHA="${{ github.sha }}" \ + # --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ + # --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \ + # --label "org.opencontainers.image.revision=${{ github.sha }}" \ + # --push \ + # -t $ECR_URL:pr-${{ github.event.pull_request.number }} \ + # -t $ECR_URL:${{ steps.build-metadata.outputs.BRANCH_NAME }} \ + # -t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \ + # -f quadratic-${{ matrix.service }}/Dockerfile . + + # - name: Move cache + # run: | + # rm -rf /tmp/.buildx-cache + # mv /tmp/.buildx-cache-new /tmp/.buildx-cache + + - name: Build, Tag, and Push Image to Amazon ECR Public env: ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} run: | - docker buildx build \ - --cache-from=type=local,src=/tmp/.buildx-cache \ - --cache-to=type=local,dest=/tmp/.buildx-cache-new \ - --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ - --build-arg GIT_SHA="${{ github.sha }}" \ - --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ - --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \ - --label "org.opencontainers.image.revision=${{ github.sha }}" \ - --push \ - -t $ECR_URL:pr-${{ github.event.pull_request.number }} \ - -t $ECR_URL:${{ steps.build-metadata.outputs.BRANCH_NAME }} \ - -t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \ - -f quadratic-${{ matrix.service }}/Dockerfile . - - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + docker build -t $ECR_URL:pr-${{ github.event.pull_request.number }} -f quadratic-${{ matrix.service }}/Dockerfile . + docker push $ECR_URL:pr-${{ github.event.pull_request.number }} From c880b630e847a0d30e7952a0e21f156b40183f82 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 03:27:17 +0530 Subject: [PATCH 09/49] use us-east-1 for public gallery --- .github/workflows/preview-branches-publish-images.yml | 2 +- .github/workflows/production-publish-images.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 6ac2415227..daed3bb44d 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -35,7 +35,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: us-east-1 - name: Login to Amazon ECR Public id: login-ecr diff --git a/.github/workflows/production-publish-images.yml b/.github/workflows/production-publish-images.yml index d7e79df8aa..79c20b0abd 100644 --- a/.github/workflows/production-publish-images.yml +++ b/.github/workflows/production-publish-images.yml @@ -22,7 +22,7 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} + aws-region: us-east-1 - name: Login to Amazon ECR Public id: login-ecr From 8fc918b227e12ca202f47a355678d2f42bfa310a Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 03:40:21 +0530 Subject: [PATCH 10/49] try 64 cores --- .github/workflows/preview-branches-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index daed3bb44d..c57fdf299a 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -10,7 +10,7 @@ concurrency: jobs: build_images: - runs-on: ubuntu-latest-8-cores + runs-on: ubuntu-latest-64-cores permissions: contents: read timeout-minutes: 10 From 6ea1712e7d464e41ae5bc0f2d89ee493cd6aa766 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 04:18:20 +0530 Subject: [PATCH 11/49] increase timeout --- .../preview-branches-publish-images.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index c57fdf299a..20465808e6 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -10,14 +10,26 @@ concurrency: jobs: build_images: - runs-on: ubuntu-latest-64-cores permissions: contents: read - timeout-minutes: 10 + + timeout-minutes: 30 + + runs-on: ${{ matrix.runner }} strategy: matrix: - service: [multiplayer, files, connection, client, api] + include: + - service: api + runner: ubuntu-latest-8-cores + - service: client + runner: ubuntu-latest-8-cores + - service: connection + runner: ubuntu-latest-8-cores + - service: files + runner: ubuntu-latest-8-cores + - service: multiplayer + runner: ubuntu-latest-8-cores fail-fast: true steps: From 24a44dc18703357cb7a822d1a82edcda5e5c7aa9 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 04:41:26 +0530 Subject: [PATCH 12/49] try 64 again --- .github/workflows/preview-branches-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 20465808e6..0cc59aea27 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -23,7 +23,7 @@ jobs: - service: api runner: ubuntu-latest-8-cores - service: client - runner: ubuntu-latest-8-cores + runner: ubuntu-latest-64-cores - service: connection runner: ubuntu-latest-8-cores - service: files From 7802f1767120fe25b14670e4d58cf6aa2b774a39 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 05:07:53 +0530 Subject: [PATCH 13/49] 4 core --- .github/workflows/preview-branches-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 0cc59aea27..ce27d0bcb0 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -23,7 +23,7 @@ jobs: - service: api runner: ubuntu-latest-8-cores - service: client - runner: ubuntu-latest-64-cores + runner: ubuntu-latest-4-cores - service: connection runner: ubuntu-latest-8-cores - service: files From 44a3861e6342482140da591294507cd02e36df4d Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 05:34:46 +0530 Subject: [PATCH 14/49] try blacksmith --- .github/workflows/preview-branches-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index ce27d0bcb0..1e9d1acd73 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -23,7 +23,7 @@ jobs: - service: api runner: ubuntu-latest-8-cores - service: client - runner: ubuntu-latest-4-cores + runner: blacksmith-8vcpu-ubuntu-2204 - service: connection runner: ubuntu-latest-8-cores - service: files From 06eed50ec74657bd0ad3b34c162a8432530f0335 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 05:52:01 +0530 Subject: [PATCH 15/49] blacksmith 32 core --- .github/workflows/preview-branches-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 1e9d1acd73..5774ddea83 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -23,7 +23,7 @@ jobs: - service: api runner: ubuntu-latest-8-cores - service: client - runner: blacksmith-8vcpu-ubuntu-2204 + runner: blacksmith-32vcpu-ubuntu-2204 - service: connection runner: ubuntu-latest-8-cores - service: files From 6395c9f5d56e26060fb94fd9198f3bd011352998 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 06:12:41 +0530 Subject: [PATCH 16/49] try changing build script --- quadratic-core/package.json | 2 +- quadratic-rust-client/package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/quadratic-core/package.json b/quadratic-core/package.json index fd52cd67ca..3b76235638 100644 --- a/quadratic-core/package.json +++ b/quadratic-core/package.json @@ -5,7 +5,7 @@ "scripts": { "start": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'", "performance": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'", - "build": "wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs", + "build": "wasm-pack build --release --target web --out-dir ../quadratic-client/src/app/quadratic-core", "coverage": "npm run coverage:clean && npm run coverage:wasm:gen && npm run coverage:wasm:html && npm run coverage:wasm:view", "coverage:wasm:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test", "coverage:wasm:html": "grcov . --binary-path ../target/debug/deps/ -s src -t html --branch --ignore-not-existing --ignore 'src/wasm_bindings/*' --ignore 'src/bin/*' --ignore '../*' --ignore '/*' -o coverage/html", diff --git a/quadratic-rust-client/package.json b/quadratic-rust-client/package.json index ae80d519d4..fbc498df49 100644 --- a/quadratic-rust-client/package.json +++ b/quadratic-rust-client/package.json @@ -2,7 +2,7 @@ "name": "quadratic-rust-client", "private": true, "scripts": { - "build": "wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs", + "build": "wasm-pack build --release --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client", "dev": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'", "dev:perf": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'" } From a6bf30f281811c58038ba0e5de7cfbe8942a1c0e Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 06:48:07 +0530 Subject: [PATCH 17/49] revert changes, add cache --- .../preview-branches-publish-images.yml | 77 ++++++------------- quadratic-core/package.json | 2 +- quadratic-rust-client/package.json | 2 +- 3 files changed, 25 insertions(+), 56 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 5774ddea83..2a2b9f871e 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -23,7 +23,7 @@ jobs: - service: api runner: ubuntu-latest-8-cores - service: client - runner: blacksmith-32vcpu-ubuntu-2204 + runner: ubuntu-latest-64-cores - service: connection runner: ubuntu-latest-8-cores - service: files @@ -47,81 +47,50 @@ jobs: with: aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }} aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} - aws-region: us-east-1 + aws-region: ${{ secrets.AWS_REGION }} - - name: Login to Amazon ECR Public + - name: Login to Amazon ECR Private id: login-ecr uses: aws-actions/amazon-ecr-login@v2 - with: - registry-type: public - name: Define repository name id: repo-name run: | echo "REPO_NAME=quadratic-${{ matrix.service }}-development" >> $GITHUB_OUTPUT - - name: Create Public ECR Repository if not exists + - name: Create Private ECR Repository if not exists id: create-ecr env: REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }} run: | # Try to describe the repository first - if ! aws ecr-public describe-repositories --repository-names $REPO_NAME 2>/dev/null; then + if ! aws ecr describe-repositories --repository-names $REPO_NAME 2>/dev/null; then # Repository doesn't exist, create it - aws ecr-public create-repository --repository-name $REPO_NAME + aws ecr create-repository --repository-name $REPO_NAME || true fi # Get the repository URI either way - REPO_INFO=$(aws ecr-public describe-repositories --repository-names $REPO_NAME) + REPO_INFO=$(aws ecr describe-repositories --repository-names $REPO_NAME) ECR_URL=$(echo $REPO_INFO | jq -r '.repositories[0].repositoryUri') echo "ECR_URL=$ECR_URL" >> $GITHUB_OUTPUT - # - name: Set up Docker Buildx - # uses: docker/setup-buildx-action@v3 - # with: - # buildkitd-flags: --debug - - # - name: Cache Docker layers - # uses: actions/cache@v3 - # with: - # path: /tmp/.buildx-cache - # key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }} - # restore-keys: | - # ${{ runner.os }}-buildx-${{ matrix.service }}- - # ${{ runner.os }}-buildx- - - # - name: Create and initialize buildx cache - # run: | - # mkdir -p /tmp/.buildx-cache - # CACHE_TAG="${{ runner.os }}-buildx-${{ matrix.service }}" - # echo "{\"layers\":{},\"manifests\":{\"$CACHE_TAG\":{\"layers\":[]}}}" > /tmp/.buildx-cache/index.json - - # - name: Build, Tag, and Push Image to Amazon ECR Private - # env: - # ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} - # run: | - # docker buildx build \ - # --cache-from=type=local,src=/tmp/.buildx-cache \ - # --cache-to=type=local,dest=/tmp/.buildx-cache-new \ - # --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ - # --build-arg GIT_SHA="${{ github.sha }}" \ - # --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ - # --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \ - # --label "org.opencontainers.image.revision=${{ github.sha }}" \ - # --push \ - # -t $ECR_URL:pr-${{ github.event.pull_request.number }} \ - # -t $ECR_URL:${{ steps.build-metadata.outputs.BRANCH_NAME }} \ - # -t $ECR_URL:${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} \ - # -f quadratic-${{ matrix.service }}/Dockerfile . - - # - name: Move cache - # run: | - # rm -rf /tmp/.buildx-cache - # mv /tmp/.buildx-cache-new /tmp/.buildx-cache + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + with: + buildkitd-flags: --debug - - name: Build, Tag, and Push Image to Amazon ECR Public + - name: Build, Tag, and Push Image to Amazon ECR Private env: ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} run: | - docker build -t $ECR_URL:pr-${{ github.event.pull_request.number }} -f quadratic-${{ matrix.service }}/Dockerfile . - docker push $ECR_URL:pr-${{ github.event.pull_request.number }} + docker buildx build \ + --cache-from=type=registry,ref=$ECR_URL:buildcache \ + --cache-to=type=registry,ref=$ECR_URL:buildcache,mode=max \ + --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ + --build-arg GIT_SHA="${{ github.sha }}" \ + --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ + --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \ + --label "org.opencontainers.image.revision=${{ github.sha }}" \ + --push \ + -t $ECR_URL:pr-${{ github.event.pull_request.number }} \ + -f quadratic-${{ matrix.service }}/Dockerfile . diff --git a/quadratic-core/package.json b/quadratic-core/package.json index 3b76235638..fd52cd67ca 100644 --- a/quadratic-core/package.json +++ b/quadratic-core/package.json @@ -5,7 +5,7 @@ "scripts": { "start": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'", "performance": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'", - "build": "wasm-pack build --release --target web --out-dir ../quadratic-client/src/app/quadratic-core", + "build": "wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs", "coverage": "npm run coverage:clean && npm run coverage:wasm:gen && npm run coverage:wasm:html && npm run coverage:wasm:view", "coverage:wasm:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test", "coverage:wasm:html": "grcov . --binary-path ../target/debug/deps/ -s src -t html --branch --ignore-not-existing --ignore 'src/wasm_bindings/*' --ignore 'src/bin/*' --ignore '../*' --ignore '/*' -o coverage/html", diff --git a/quadratic-rust-client/package.json b/quadratic-rust-client/package.json index fbc498df49..ae80d519d4 100644 --- a/quadratic-rust-client/package.json +++ b/quadratic-rust-client/package.json @@ -2,7 +2,7 @@ "name": "quadratic-rust-client", "private": true, "scripts": { - "build": "wasm-pack build --release --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client", + "build": "wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs", "dev": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'", "dev:perf": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-rust-client --weak-refs'" } From ce65f365762eab5a80a85a7c2f659f4579c83554 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 07:00:07 +0530 Subject: [PATCH 18/49] try --- .github/workflows/preview-branches-publish-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 2a2b9f871e..94ac2bcf5c 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -84,8 +84,8 @@ jobs: ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} run: | docker buildx build \ - --cache-from=type=registry,ref=$ECR_URL:buildcache \ - --cache-to=type=registry,ref=$ECR_URL:buildcache,mode=max \ + # --cache-from=type=registry,ref=$ECR_URL:buildcache \ + # --cache-to=type=registry,ref=$ECR_URL:buildcache,mode=max \ --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ --build-arg GIT_SHA="${{ github.sha }}" \ --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ From cb04f4bde0d31579abca82ad387f71ab219a91b1 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 07:04:23 +0530 Subject: [PATCH 19/49] try --- .github/workflows/preview-branches-publish-images.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 94ac2bcf5c..5c5b56b0c9 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -84,8 +84,6 @@ jobs: ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} run: | docker buildx build \ - # --cache-from=type=registry,ref=$ECR_URL:buildcache \ - # --cache-to=type=registry,ref=$ECR_URL:buildcache,mode=max \ --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ --build-arg GIT_SHA="${{ github.sha }}" \ --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ @@ -93,4 +91,5 @@ jobs: --label "org.opencontainers.image.revision=${{ github.sha }}" \ --push \ -t $ECR_URL:pr-${{ github.event.pull_request.number }} \ - -f quadratic-${{ matrix.service }}/Dockerfile . + -f quadratic-${{ matrix.service }}/Dockerfile \ + . From 7dc5f3c5331c6162f831190de2e9a481a2c2018e Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 07:25:24 +0530 Subject: [PATCH 20/49] try caching again --- .../preview-branches-publish-images.yml | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 5c5b56b0c9..3fd50325be 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -79,11 +79,23 @@ jobs: with: buildkitd-flags: --debug + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} + restore-keys: | + ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} + ${{ runner.os }}-buildx-${{ matrix.service }}- + ${{ runner.os }}-buildx- + - name: Build, Tag, and Push Image to Amazon ECR Private env: ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} run: | docker buildx build \ + --cache-from=type=local,src=/tmp/.buildx-cache \ + --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \ --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ --build-arg GIT_SHA="${{ github.sha }}" \ --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ @@ -93,3 +105,11 @@ jobs: -t $ECR_URL:pr-${{ github.event.pull_request.number }} \ -f quadratic-${{ matrix.service }}/Dockerfile \ . + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache From 5e04cf7471e9c63a357ea099e1d6daffd82465fc Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 09:16:06 +0530 Subject: [PATCH 21/49] build image caching done --- .github/workflows/preview-branches-publish-images.yml | 4 ++-- package.json | 1 - quadratic-api/Dockerfile | 2 +- quadratic-client/Dockerfile | 10 ++++------ 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 3fd50325be..260ae7c2e6 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -23,7 +23,7 @@ jobs: - service: api runner: ubuntu-latest-8-cores - service: client - runner: ubuntu-latest-64-cores + runner: ubuntu-latest-8-cores - service: connection runner: ubuntu-latest-8-cores - service: files @@ -85,7 +85,6 @@ jobs: path: /tmp/.buildx-cache key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} restore-keys: | - ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} ${{ runner.os }}-buildx-${{ matrix.service }}- ${{ runner.os }}-buildx- @@ -96,6 +95,7 @@ jobs: docker buildx build \ --cache-from=type=local,src=/tmp/.buildx-cache \ --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ --build-arg GIT_SHA="${{ github.sha }}" \ --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ diff --git a/package.json b/package.json index d685418d0a..55421dbd3c 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,6 @@ }, "dependencies": { "@ory/kratos-client": "^1.2.1", - "tsc": "^2.0.4", "vitest": "^1.5.0", "zod": "^3.23.8" }, diff --git a/quadratic-api/Dockerfile b/quadratic-api/Dockerfile index 77a8ba4e16..ff5c16668d 100644 --- a/quadratic-api/Dockerfile +++ b/quadratic-api/Dockerfile @@ -2,9 +2,9 @@ FROM node:18-alpine AS builder WORKDIR /app COPY package.json . COPY package-lock.json . +RUN npm install COPY quadratic-api ./quadratic-api COPY quadratic-shared ./quadratic-shared -RUN npm install FROM node:18-slim AS runtime WORKDIR /app diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index e814a28455..ac575c408a 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -12,8 +12,8 @@ RUN echo 'wasm-pack version:' && wasm-pack --version # Install wasm32-unknown-unknown target # RUN rustup target add wasm32-unknown-unknown -# Install python & clean up -RUN apt-get update && apt-get install -y python-is-python3 python3-pip && apt-get clean && rm -rf /var/lib/apt/lists/* +# Install python +RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip # Install npm dependencies WORKDIR /app @@ -24,10 +24,7 @@ COPY ./quadratic-core/package*.json ./quadratic-core/ COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/ COPY ./quadratic-shared/package*.json ./quadratic-shared/ COPY ./quadratic-client/package*.json ./quadratic-client/ -RUN npm install - -# Install typescript -RUN npm install -D typescript +RUN npm install --no-audit --no-fund # Copy the rest of the application WORKDIR /app @@ -51,6 +48,7 @@ WORKDIR /app/quadratic-core RUN echo 'Exporting TS/Rust types...' && cargo run --bin export_types # Build the quadratic-rust-client +# Cache will not work beyond this point on CI because of GIT_COMMIT being different for each commit WORKDIR /app ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT From 23413df363bca7565a1752e93f5cfd23a1e99097 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 09:22:23 +0530 Subject: [PATCH 22/49] fix api dockerfile --- quadratic-api/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/quadratic-api/Dockerfile b/quadratic-api/Dockerfile index ff5c16668d..330ea76583 100644 --- a/quadratic-api/Dockerfile +++ b/quadratic-api/Dockerfile @@ -2,6 +2,7 @@ FROM node:18-alpine AS builder WORKDIR /app COPY package.json . COPY package-lock.json . +COPY ./quadratic-api/package*.json ./quadratic-api/ RUN npm install COPY quadratic-api ./quadratic-api COPY quadratic-shared ./quadratic-shared From bbe0caba73f5e3f30dac8ae194a5da85485b72a3 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 09:29:47 +0530 Subject: [PATCH 23/49] really fix api dockerfile --- quadratic-api/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/quadratic-api/Dockerfile b/quadratic-api/Dockerfile index 330ea76583..69bec4f4ae 100644 --- a/quadratic-api/Dockerfile +++ b/quadratic-api/Dockerfile @@ -3,6 +3,7 @@ WORKDIR /app COPY package.json . COPY package-lock.json . COPY ./quadratic-api/package*.json ./quadratic-api/ +COPY ./quadratic-shared/package*.json ./quadratic-shared/ RUN npm install COPY quadratic-api ./quadratic-api COPY quadratic-shared ./quadratic-shared From 9d00b875044e3f24a676c8793dd418a9154beb15 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 11:57:36 +0530 Subject: [PATCH 24/49] try BuildKit inline cache --- quadratic-client/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index ac575c408a..3a65ef0954 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -24,7 +24,8 @@ COPY ./quadratic-core/package*.json ./quadratic-core/ COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/ COPY ./quadratic-shared/package*.json ./quadratic-shared/ COPY ./quadratic-client/package*.json ./quadratic-client/ -RUN npm install --no-audit --no-fund +RUN --mount=type=cache,target=/root/.npm \ + npm install --no-audit --no-fund # Copy the rest of the application WORKDIR /app From 1f61596e2d0f486db6846b4fc9570441ae8569c1 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sat, 21 Dec 2024 12:19:10 +0530 Subject: [PATCH 25/49] more cache trial and blacksmith 2 core --- .../preview-branches-publish-images.yml | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 260ae7c2e6..47c4bfc56f 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -23,7 +23,7 @@ jobs: - service: api runner: ubuntu-latest-8-cores - service: client - runner: ubuntu-latest-8-cores + runner: blacksmith-2vcpu-ubuntu-2204 - service: connection runner: ubuntu-latest-8-cores - service: files @@ -77,7 +77,9 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 with: - buildkitd-flags: --debug + driver-opts: | + image=moby/buildkit:latest + network=host - name: Cache Docker layers uses: actions/cache@v3 @@ -88,23 +90,27 @@ jobs: ${{ runner.os }}-buildx-${{ matrix.service }}- ${{ runner.os }}-buildx- - - name: Build, Tag, and Push Image to Amazon ECR Private - env: - ECR_URL: ${{ steps.create-ecr.outputs.ECR_URL }} - run: | - docker buildx build \ - --cache-from=type=local,src=/tmp/.buildx-cache \ - --cache-to=type=local,dest=/tmp/.buildx-cache-new,mode=max \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - --build-arg BUILD_TIME="${{ steps.build-metadata.outputs.BUILD_TIME }}" \ - --build-arg GIT_SHA="${{ github.sha }}" \ - --build-arg PR_NUMBER="${{ github.event.pull_request.number }}" \ - --label "org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }}" \ - --label "org.opencontainers.image.revision=${{ github.sha }}" \ - --push \ - -t $ECR_URL:pr-${{ github.event.pull_request.number }} \ - -f quadratic-${{ matrix.service }}/Dockerfile \ - . + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: quadratic-${{ matrix.service }}/Dockerfile + push: true + tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} + cache-from: | + type=local,src=/tmp/.buildx-cache + type=registry,ref=${{ steps.create-ecr.outputs.ECR_URL }}:buildcache + cache-to: | + type=local,dest=/tmp/.buildx-cache-new,mode=max + type=registry,ref=${{ steps.create-ecr.outputs.ECR_URL }}:buildcache,mode=max + build-args: | + BUILDKIT_INLINE_CACHE=1 + BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} + GIT_SHA=${{ github.sha }} + PR_NUMBER=${{ github.event.pull_request.number }} + labels: | + org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }} + org.opencontainers.image.revision=${{ github.sha }} # Temp fix # https://github.com/docker/build-push-action/issues/252 From 49bd52d5b1a272b758150eb09f3d259d85115799 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 00:50:52 +0530 Subject: [PATCH 26/49] reduce docker layers, use package scripts --- .../preview-branches-publish-images.yml | 34 ++++++++----------- package.json | 1 + quadratic-client/Dockerfile | 27 +++++---------- quadratic-core/package.json | 3 +- 4 files changed, 27 insertions(+), 38 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 47c4bfc56f..67777b9ee5 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -23,7 +23,7 @@ jobs: - service: api runner: ubuntu-latest-8-cores - service: client - runner: blacksmith-2vcpu-ubuntu-2204 + runner: blacksmith-4vcpu-ubuntu-2204 - service: connection runner: ubuntu-latest-8-cores - service: files @@ -81,14 +81,14 @@ jobs: image=moby/buildkit:latest network=host - - name: Cache Docker layers - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} - restore-keys: | - ${{ runner.os }}-buildx-${{ matrix.service }}- - ${{ runner.os }}-buildx- + # - name: Cache Docker layers + # uses: actions/cache@v3 + # with: + # path: /tmp/.buildx-cache + # key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} + # restore-keys: | + # ${{ runner.os }}-buildx-${{ matrix.service }}- + # ${{ runner.os }}-buildx- - name: Build and push uses: docker/build-push-action@v6 @@ -97,12 +97,8 @@ jobs: file: quadratic-${{ matrix.service }}/Dockerfile push: true tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} - cache-from: | - type=local,src=/tmp/.buildx-cache - type=registry,ref=${{ steps.create-ecr.outputs.ECR_URL }}:buildcache - cache-to: | - type=local,dest=/tmp/.buildx-cache-new,mode=max - type=registry,ref=${{ steps.create-ecr.outputs.ECR_URL }}:buildcache,mode=max + # cache-from: type=gha,scope=${{ matrix.service }} + # cache-to: type=gha,mode=max,scope=${{ matrix.service }} build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} @@ -115,7 +111,7 @@ jobs: # Temp fix # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + # - name: Move cache + # run: | + # rm -rf /tmp/.buildx-cache + # mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/package.json b/package.json index 55421dbd3c..180cfea028 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "quadratic-files", "quadratic-multiplayer", "quadratic-rust-client", + "quadratic-core", "quadratic-client", "quadratic-kernels/python-wasm" ], diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index 3a65ef0954..260dc156ad 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -15,8 +15,9 @@ RUN echo 'wasm-pack version:' && wasm-pack --version # Install python RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip -# Install npm dependencies WORKDIR /app + +# Copy all package.json files COPY package.json . COPY package-lock.json . COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/ @@ -24,11 +25,11 @@ COPY ./quadratic-core/package*.json ./quadratic-core/ COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/ COPY ./quadratic-shared/package*.json ./quadratic-shared/ COPY ./quadratic-client/package*.json ./quadratic-client/ -RUN --mount=type=cache,target=/root/.npm \ - npm install --no-audit --no-fund + +# Install npm dependencies +RUN npm install --no-audit --no-fund # Copy the rest of the application -WORKDIR /app COPY updateAlertVersion.json . COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/ COPY ./quadratic-core/. ./quadratic-core/ @@ -37,38 +38,28 @@ COPY ./quadratic-shared/. ./quadratic-shared/ COPY ./quadratic-client/. ./quadratic-client/ # Run the packaging script for quadratic_py -WORKDIR /app RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry -# Build wasm -WORKDIR /app/quadratic-core -RUN echo 'Building wasm...' && wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs - -# Export TS/Rust types -WORKDIR /app/quadratic-core -RUN echo 'Exporting TS/Rust types...' && cargo run --bin export_types +# Build wasm and export TS/Rust types +RUN echo 'Building wasm...' && npm run build --workspace=quadratic-core # Build the quadratic-rust-client -# Cache will not work beyond this point on CI because of GIT_COMMIT being different for each commit -WORKDIR /app +# Layer caching will not work beyond this point on CI because of GIT_COMMIT being different for each commit ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client # Build the quadratic-shared -WORKDIR /app RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared # Build the front-end -WORKDIR /app -RUN echo 'Building front-end...' ENV VITE_DEBUG=VITE_DEBUG_VAL ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL ENV VITE_QUADRATIC_CONNECTION_URL=VITE_QUADRATIC_CONNECTION_URL_VAL ENV VITE_AUTH_TYPE=VITE_AUTH_TYPE_VAL ENV VITE_ORY_HOST=VITE_ORY_HOST_VAL -RUN npm run build --workspace=quadratic-client +RUN echo 'Building front-end...' && npm run build --workspace=quadratic-client # The default command to run the application # CMD ["npm", "run", "start:production"] diff --git a/quadratic-core/package.json b/quadratic-core/package.json index fd52cd67ca..0af92d0167 100644 --- a/quadratic-core/package.json +++ b/quadratic-core/package.json @@ -5,7 +5,8 @@ "scripts": { "start": "cargo watch -s 'wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'", "performance": "cargo watch -s 'wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs'", - "build": "wasm-pack build --dev --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs", + "build": "wasm-pack build --target web --out-dir ../quadratic-client/src/app/quadratic-core --weak-refs", + "postbuild": "cargo run --bin export_types --features js", "coverage": "npm run coverage:clean && npm run coverage:wasm:gen && npm run coverage:wasm:html && npm run coverage:wasm:view", "coverage:wasm:gen": "CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='coverage/cargo-test-%p-%m.profraw' cargo test", "coverage:wasm:html": "grcov . --binary-path ../target/debug/deps/ -s src -t html --branch --ignore-not-existing --ignore 'src/wasm_bindings/*' --ignore 'src/bin/*' --ignore '../*' --ignore '/*' -o coverage/html", From 167c7024523006d513bec2fd1d3725d7d17387e0 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 01:36:21 +0530 Subject: [PATCH 27/49] try blacksmith 2 core for other services --- .github/workflows/preview-branches-publish-images.yml | 10 +++++----- quadratic-client/Dockerfile | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 67777b9ee5..9f5125dcc4 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -21,15 +21,15 @@ jobs: matrix: include: - service: api - runner: ubuntu-latest-8-cores + runner: blacksmith-2vcpu-ubuntu-2204 - service: client - runner: blacksmith-4vcpu-ubuntu-2204 + runner: blacksmith-8vcpu-ubuntu-2204 - service: connection - runner: ubuntu-latest-8-cores + runner: blacksmith-2vcpu-ubuntu-2204 - service: files - runner: ubuntu-latest-8-cores + runner: blacksmith-2vcpu-ubuntu-2204 - service: multiplayer - runner: ubuntu-latest-8-cores + runner: blacksmith-2vcpu-ubuntu-2204 fail-fast: true steps: diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index 260dc156ad..f765d3529c 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -4,13 +4,17 @@ FROM node:18 AS build # Install rustup RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" +ENV CARGO_TARGET_DIR=/app/target +ENV CARGO_HOME=/app/.cargo +ENV CARGO_BUILD_JOBS=64 +ENV RUSTFLAGS='-C codegen-units=64' # Install wasm-pack RUN echo 'Installing wasm-pack...' && curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh RUN echo 'wasm-pack version:' && wasm-pack --version # Install wasm32-unknown-unknown target -# RUN rustup target add wasm32-unknown-unknown +RUN rustup target add wasm32-unknown-unknown # Install python RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip From 7b41e43a0613ebb8b90e942527903b1c6df72889 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 02:30:01 +0530 Subject: [PATCH 28/49] try blacksmith cache --- .../preview-branches-publish-images.yml | 30 +++++++++---------- .vscode/settings.json | 5 ++++ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 9f5125dcc4..599363c544 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -23,7 +23,7 @@ jobs: - service: api runner: blacksmith-2vcpu-ubuntu-2204 - service: client - runner: blacksmith-8vcpu-ubuntu-2204 + runner: blacksmith-4vcpu-ubuntu-2204 # no speed benefit of 4vcpu, 2vcpu runs out of memory - service: connection runner: blacksmith-2vcpu-ubuntu-2204 - service: files @@ -81,14 +81,14 @@ jobs: image=moby/buildkit:latest network=host - # - name: Cache Docker layers - # uses: actions/cache@v3 - # with: - # path: /tmp/.buildx-cache - # key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} - # restore-keys: | - # ${{ runner.os }}-buildx-${{ matrix.service }}- - # ${{ runner.os }}-buildx- + - name: Cache Docker layers + uses: useblacksmith/cache@v5 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} + restore-keys: | + ${{ runner.os }}-buildx-${{ matrix.service }}- + ${{ runner.os }}-buildx- - name: Build and push uses: docker/build-push-action@v6 @@ -97,8 +97,8 @@ jobs: file: quadratic-${{ matrix.service }}/Dockerfile push: true tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} - # cache-from: type=gha,scope=${{ matrix.service }} - # cache-to: type=gha,mode=max,scope=${{ matrix.service }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} @@ -111,7 +111,7 @@ jobs: # Temp fix # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 - # - name: Move cache - # run: | - # rm -rf /tmp/.buildx-cache - # mv /tmp/.buildx-cache-new /tmp/.buildx-cache + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.vscode/settings.json b/.vscode/settings.json index 1d8a0dafbb..1d4922aaef 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,6 +6,8 @@ "bigdecimal", "bincode", "bindgen", + "buildkit", + "Buildx", "CRPXNLSKVLJFHH", "dashmap", "dbgjs", @@ -32,6 +34,7 @@ "MDSL", "micropip", "minmax", + "moby", "msdf", "nonblank", "Northbridge", @@ -59,7 +62,9 @@ "trackpad", "undoable", "unspill", + "useblacksmith", "vals", + "vcpu", "websockets", "Westborough" ], From f2a27f2c1a3b9aa83c0ccb989644b96a35db9007 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 03:01:52 +0530 Subject: [PATCH 29/49] add cache compression --- .github/workflows/preview-branches-publish-images.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 599363c544..e2cfb2bccb 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -81,14 +81,13 @@ jobs: image=moby/buildkit:latest network=host - - name: Cache Docker layers + - name: Set up Docker layer cache uses: useblacksmith/cache@v5 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} + key: buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} restore-keys: | - ${{ runner.os }}-buildx-${{ matrix.service }}- - ${{ runner.os }}-buildx- + buildx-${{ matrix.service }}- - name: Build and push uses: docker/build-push-action@v6 @@ -98,7 +97,7 @@ jobs: push: true tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd,compression-level=22,force-compression=true build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} @@ -111,7 +110,7 @@ jobs: # Temp fix # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 - - name: Move cache + - name: Update cache run: | rm -rf /tmp/.buildx-cache mv /tmp/.buildx-cache-new /tmp/.buildx-cache From 3e137c46cce43fc00a08498cbe2932c92bdfaeaf Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 03:25:40 +0530 Subject: [PATCH 30/49] try default compression level --- .github/workflows/preview-branches-publish-images.yml | 2 +- .vscode/settings.json | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index e2cfb2bccb..931902091e 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -97,7 +97,7 @@ jobs: push: true tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd,compression-level=22,force-compression=true + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} diff --git a/.vscode/settings.json b/.vscode/settings.json index 1d4922aaef..f0d8987b2b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -66,7 +66,8 @@ "vals", "vcpu", "websockets", - "Westborough" + "Westborough", + "zstd" ], "editor.codeActionsOnSave": { "source.organizeImports": "explicit" From 92a8728a3d8097b876c42ee0a10eb126d81d7ac7 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 03:49:03 +0530 Subject: [PATCH 31/49] revert to max cache compression --- .github/workflows/preview-branches-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 931902091e..e2cfb2bccb 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -97,7 +97,7 @@ jobs: push: true tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd,compression-level=22,force-compression=true build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} From 1473500b432b7c2659bbd62d1afaa7a2b941adb4 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 04:46:40 +0530 Subject: [PATCH 32/49] reduce build layer of connections, files and multiplayer --- .dockerignore | 43 ++++++++++++++++++++++++++++- Cargo.lock | 1 - quadratic-api/.dockerignore | 43 +++++++++++++++++++++++++++++ quadratic-client/.dockerignore | 43 +++++++++++++++++++++++++++++ quadratic-client/Dockerfile | 8 +++--- quadratic-connection/.dockerignore | 43 +++++++++++++++++++++++++++++ quadratic-connection/Cargo.toml | 34 ++++++++--------------- quadratic-connection/Dockerfile | 16 ++++++----- quadratic-files/.dockerignore | 43 +++++++++++++++++++++++++++++ quadratic-files/Dockerfile | 20 ++++++++------ quadratic-multiplayer/.dockerignore | 43 +++++++++++++++++++++++++++++ quadratic-multiplayer/Dockerfile | 19 ++++++++----- 12 files changed, 305 insertions(+), 51 deletions(-) create mode 100644 quadratic-api/.dockerignore create mode 100644 quadratic-client/.dockerignore create mode 100644 quadratic-connection/.dockerignore create mode 100644 quadratic-files/.dockerignore create mode 100644 quadratic-multiplayer/.dockerignore diff --git a/.dockerignore b/.dockerignore index 1f5f855728..5b5eab44ef 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,43 @@ +# Environment files +.env* +**/.env* + +# Docker files +.dockerignore +**/.dockerignore +Dockerfile* +**/Dockerfile* +docker-compose* +**/docker-compose* + +# Build outputs +build/ +**/build/ +dist/ +**/dist/ target/ -**/target/ \ No newline at end of file +**/target/ + +# Dependencies +node_modules/ +**/node_modules/ +vendor/ +**/vendor/ + +# Test and coverage +coverage/ +**/coverage/ +*.test.* +**/*.test.* + +# Version control +.git/ +.gitignore +.gitattributes + +# IDE and editor files +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 82e4ccbc2e..c06d42eaa6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4501,7 +4501,6 @@ dependencies = [ "log", "openssl", "parquet 51.0.0", - "quadratic-core", "quadratic-rust-shared", "reqwest 0.11.27", "serde", diff --git a/quadratic-api/.dockerignore b/quadratic-api/.dockerignore new file mode 100644 index 0000000000..5b5eab44ef --- /dev/null +++ b/quadratic-api/.dockerignore @@ -0,0 +1,43 @@ +# Environment files +.env* +**/.env* + +# Docker files +.dockerignore +**/.dockerignore +Dockerfile* +**/Dockerfile* +docker-compose* +**/docker-compose* + +# Build outputs +build/ +**/build/ +dist/ +**/dist/ +target/ +**/target/ + +# Dependencies +node_modules/ +**/node_modules/ +vendor/ +**/vendor/ + +# Test and coverage +coverage/ +**/coverage/ +*.test.* +**/*.test.* + +# Version control +.git/ +.gitignore +.gitattributes + +# IDE and editor files +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store \ No newline at end of file diff --git a/quadratic-client/.dockerignore b/quadratic-client/.dockerignore new file mode 100644 index 0000000000..5b5eab44ef --- /dev/null +++ b/quadratic-client/.dockerignore @@ -0,0 +1,43 @@ +# Environment files +.env* +**/.env* + +# Docker files +.dockerignore +**/.dockerignore +Dockerfile* +**/Dockerfile* +docker-compose* +**/docker-compose* + +# Build outputs +build/ +**/build/ +dist/ +**/dist/ +target/ +**/target/ + +# Dependencies +node_modules/ +**/node_modules/ +vendor/ +**/vendor/ + +# Test and coverage +coverage/ +**/coverage/ +*.test.* +**/*.test.* + +# Version control +.git/ +.gitignore +.gitattributes + +# IDE and editor files +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store \ No newline at end of file diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index f765d3529c..8c2dfd58d2 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -4,8 +4,8 @@ FROM node:18 AS build # Install rustup RUN echo 'Installing rustup...' && curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" -ENV CARGO_TARGET_DIR=/app/target -ENV CARGO_HOME=/app/.cargo +ENV CARGO_TARGET_DIR=/quadratic/target +ENV CARGO_HOME=/quadratic/.cargo ENV CARGO_BUILD_JOBS=64 ENV RUSTFLAGS='-C codegen-units=64' @@ -19,7 +19,7 @@ RUN rustup target add wasm32-unknown-unknown # Install python RUN apt-get update && apt-get install -y --no-install-recommends python-is-python3 python3-pip -WORKDIR /app +WORKDIR /quadratic # Copy all package.json files COPY package.json . @@ -69,7 +69,7 @@ RUN echo 'Building front-end...' && npm run build --workspace=quadratic-client # CMD ["npm", "run", "start:production"] FROM nginx:stable-alpine -COPY --from=build /app/build /usr/share/nginx/html +COPY --from=build /quadratic/build /usr/share/nginx/html EXPOSE 80 443 3000 diff --git a/quadratic-connection/.dockerignore b/quadratic-connection/.dockerignore new file mode 100644 index 0000000000..5b5eab44ef --- /dev/null +++ b/quadratic-connection/.dockerignore @@ -0,0 +1,43 @@ +# Environment files +.env* +**/.env* + +# Docker files +.dockerignore +**/.dockerignore +Dockerfile* +**/Dockerfile* +docker-compose* +**/docker-compose* + +# Build outputs +build/ +**/build/ +dist/ +**/dist/ +target/ +**/target/ + +# Dependencies +node_modules/ +**/node_modules/ +vendor/ +**/vendor/ + +# Test and coverage +coverage/ +**/coverage/ +*.test.* +**/*.test.* + +# Version control +.git/ +.gitignore +.gitattributes + +# IDE and editor files +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store \ No newline at end of file diff --git a/quadratic-connection/Cargo.toml b/quadratic-connection/Cargo.toml index 601908079d..f138d0899f 100644 --- a/quadratic-connection/Cargo.toml +++ b/quadratic-connection/Cargo.toml @@ -15,10 +15,7 @@ chrono = { version = "0.4.31", features = ["serde"] } dotenv = "0.15.0" envy = "0.4.2" futures = "0.3.29" -futures-util = { version = "0.3.29", default-features = false, features = [ - "sink", - "std", -] } +futures-util = { version = "0.3.29", default-features = false, features = ["sink", "std"] } headers = "0.4.0" http = "1.1.0" http-body-util = "0.1.1" @@ -27,17 +24,9 @@ hyper-util = { version = "0.1.5", features = ["service"] } jsonwebtoken = "9.2.0" log = "0.4.21" openssl = { version = "0.10.66", features = ["vendored"] } -parquet = { version = "51.0.0", default-features = false, features = [ - "arrow", - "arrow-array", -] } +parquet = { version = "51.0.0", default-features = false, features = ["arrow", "arrow-array"] } quadratic-rust-shared = { path = "../quadratic-rust-shared" } -reqwest = { version = "0.11.22", features = [ - "cookies", - "json", - "serde_json", - "stream", -] } +reqwest = { version = "0.11.22", features = ["cookies", "json", "serde_json", "stream"] } serde = { version = "1.0.193", features = ["derive"] } serde_json = "1.0.108" strum = "0.26.3" @@ -46,14 +35,14 @@ thiserror = "1.0.50" tokio = { version = "1.34.0", features = ["full"] } tower = { version = "0.4.13", features = ["util"] } tower-http = { version = "0.5.0", features = [ - "auth", - "compression-gzip", - "cors", - "fs", - "sensitive-headers", - "trace", - "util", - "validate-request", + "auth", + "compression-gzip", + "cors", + "fs", + "sensitive-headers", + "trace", + "util", + "validate-request", ] } tracing = "0.1.40" tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } @@ -61,7 +50,6 @@ uuid = { version = "1.6.1", features = ["serde", "v4"] } [dev-dependencies] fake = { version = "2.9.1", features = ["derive"] } -quadratic-core = { path = "../quadratic-core" } quadratic-rust-shared = { path = "../quadratic-rust-shared", features = ["test"] } tracing-test = "0.2.4" diff --git a/quadratic-connection/Dockerfile b/quadratic-connection/Dockerfile index 73556ef906..832306702f 100644 --- a/quadratic-connection/Dockerfile +++ b/quadratic-connection/Dockerfile @@ -1,12 +1,14 @@ -FROM rust:latest as builder +FROM rust:latest AS builder -RUN USER=root cargo new --bin quadratic-connection -COPY . /quadratic-connection +WORKDIR /quadratic +COPY ./quadratic-connection/. ./quadratic-connection/ +COPY ./quadratic-rust-shared/. ./quadratic-rust-shared/ RUN rustup component add rustfmt -WORKDIR /quadratic-connection -RUN cargo build --release --package quadratic-connection -FROM debian:stable-slim as runtime -COPY --from=builder /quadratic-connection/target/release/quadratic-connection . +WORKDIR /quadratic/quadratic-connection +RUN cargo build --release + +FROM debian:stable-slim AS runtime +COPY --from=builder /quadratic/quadratic-connection/target/release/quadratic-connection . RUN apt-get update && apt install -y ca-certificates CMD ["./quadratic-connection"] diff --git a/quadratic-files/.dockerignore b/quadratic-files/.dockerignore new file mode 100644 index 0000000000..5b5eab44ef --- /dev/null +++ b/quadratic-files/.dockerignore @@ -0,0 +1,43 @@ +# Environment files +.env* +**/.env* + +# Docker files +.dockerignore +**/.dockerignore +Dockerfile* +**/Dockerfile* +docker-compose* +**/docker-compose* + +# Build outputs +build/ +**/build/ +dist/ +**/dist/ +target/ +**/target/ + +# Dependencies +node_modules/ +**/node_modules/ +vendor/ +**/vendor/ + +# Test and coverage +coverage/ +**/coverage/ +*.test.* +**/*.test.* + +# Version control +.git/ +.gitignore +.gitattributes + +# IDE and editor files +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store \ No newline at end of file diff --git a/quadratic-files/Dockerfile b/quadratic-files/Dockerfile index 723ca8660e..de44b5c073 100644 --- a/quadratic-files/Dockerfile +++ b/quadratic-files/Dockerfile @@ -1,12 +1,16 @@ -FROM rust:latest as builder +FROM rust:latest AS builder -RUN USER=root cargo new --bin quadratic-files -COPY . /quadratic-files +WORKDIR /quadratic +COPY ./quadratic-files/. ./quadratic-files/ +COPY ./quadratic-core/. ./quadratic-core/ +COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts +COPY ./quadratic-rust-shared/. ./quadratic-rust-shared/ RUN rustup component add rustfmt -WORKDIR /quadratic-files -RUN cargo build --release --package quadratic-files -FROM debian:stable-slim as runtime -COPY --from=builder /quadratic-files/target/release/quadratic-files . +WORKDIR /quadratic/quadratic-files +RUN cargo build --release + +FROM debian:stable-slim AS runtime +COPY --from=builder /quadratic/quadratic-files/target/release/quadratic-files . RUN apt-get update && apt install -y ca-certificates -CMD ["./quadratic-files"] +CMD ["./quadratic-files"] \ No newline at end of file diff --git a/quadratic-multiplayer/.dockerignore b/quadratic-multiplayer/.dockerignore new file mode 100644 index 0000000000..5b5eab44ef --- /dev/null +++ b/quadratic-multiplayer/.dockerignore @@ -0,0 +1,43 @@ +# Environment files +.env* +**/.env* + +# Docker files +.dockerignore +**/.dockerignore +Dockerfile* +**/Dockerfile* +docker-compose* +**/docker-compose* + +# Build outputs +build/ +**/build/ +dist/ +**/dist/ +target/ +**/target/ + +# Dependencies +node_modules/ +**/node_modules/ +vendor/ +**/vendor/ + +# Test and coverage +coverage/ +**/coverage/ +*.test.* +**/*.test.* + +# Version control +.git/ +.gitignore +.gitattributes + +# IDE and editor files +.idea/ +.vscode/ +*.swp +*.swo +.DS_Store \ No newline at end of file diff --git a/quadratic-multiplayer/Dockerfile b/quadratic-multiplayer/Dockerfile index d2921d58ae..5005cf49b1 100644 --- a/quadratic-multiplayer/Dockerfile +++ b/quadratic-multiplayer/Dockerfile @@ -1,12 +1,17 @@ -FROM rust:latest as builder +FROM rust:latest AS builder -RUN USER=root cargo new --bin quadratic-multiplayer -COPY . /quadratic-multiplayer +WORKDIR /quadratic +COPY updateAlertVersion.json . +COPY ./quadratic-multiplayer/. ./quadratic-multiplayer/ +COPY ./quadratic-core/. ./quadratic-core/ +COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts +COPY ./quadratic-rust-shared/. ./quadratic-rust-shared/ RUN rustup component add rustfmt -WORKDIR /quadratic-multiplayer -RUN cargo build --release --package quadratic-multiplayer -FROM debian:stable-slim as runtime -COPY --from=builder /quadratic-multiplayer/target/release/quadratic-multiplayer . +WORKDIR /quadratic/quadratic-multiplayer +RUN cargo build --release + +FROM debian:stable-slim AS runtime +COPY --from=builder /quadratic/quadratic-multiplayer/target/release/quadratic-multiplayer . RUN apt-get update && apt install -y ca-certificates CMD ["./quadratic-multiplayer"] From 3cd98ee8a15c74b3a434e07ab9c320e84a28518e Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 07:25:45 +0530 Subject: [PATCH 33/49] fix docker ignore --- .dockerignore | 6 --- .../preview-branches-publish-images.yml | 2 +- quadratic-api/.dockerignore | 43 ------------------- quadratic-api/Dockerfile | 6 +-- quadratic-client/.dockerignore | 43 ------------------- quadratic-connection/.dockerignore | 43 ------------------- quadratic-files/.dockerignore | 43 ------------------- quadratic-multiplayer/.dockerignore | 43 ------------------- 8 files changed, 4 insertions(+), 225 deletions(-) delete mode 100644 quadratic-api/.dockerignore delete mode 100644 quadratic-client/.dockerignore delete mode 100644 quadratic-connection/.dockerignore delete mode 100644 quadratic-files/.dockerignore delete mode 100644 quadratic-multiplayer/.dockerignore diff --git a/.dockerignore b/.dockerignore index 5b5eab44ef..f95554c258 100644 --- a/.dockerignore +++ b/.dockerignore @@ -24,12 +24,6 @@ node_modules/ vendor/ **/vendor/ -# Test and coverage -coverage/ -**/coverage/ -*.test.* -**/*.test.* - # Version control .git/ .gitignore diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index e2cfb2bccb..931902091e 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -97,7 +97,7 @@ jobs: push: true tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd,compression-level=22,force-compression=true + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} diff --git a/quadratic-api/.dockerignore b/quadratic-api/.dockerignore deleted file mode 100644 index 5b5eab44ef..0000000000 --- a/quadratic-api/.dockerignore +++ /dev/null @@ -1,43 +0,0 @@ -# Environment files -.env* -**/.env* - -# Docker files -.dockerignore -**/.dockerignore -Dockerfile* -**/Dockerfile* -docker-compose* -**/docker-compose* - -# Build outputs -build/ -**/build/ -dist/ -**/dist/ -target/ -**/target/ - -# Dependencies -node_modules/ -**/node_modules/ -vendor/ -**/vendor/ - -# Test and coverage -coverage/ -**/coverage/ -*.test.* -**/*.test.* - -# Version control -.git/ -.gitignore -.gitattributes - -# IDE and editor files -.idea/ -.vscode/ -*.swp -*.swo -.DS_Store \ No newline at end of file diff --git a/quadratic-api/Dockerfile b/quadratic-api/Dockerfile index 69bec4f4ae..8b57168303 100644 --- a/quadratic-api/Dockerfile +++ b/quadratic-api/Dockerfile @@ -1,5 +1,5 @@ FROM node:18-alpine AS builder -WORKDIR /app +WORKDIR /quadratic COPY package.json . COPY package-lock.json . COPY ./quadratic-api/package*.json ./quadratic-api/ @@ -9,8 +9,8 @@ COPY quadratic-api ./quadratic-api COPY quadratic-shared ./quadratic-shared FROM node:18-slim AS runtime -WORKDIR /app -COPY --from=builder /app . +WORKDIR /quadratic +COPY --from=builder /quadratic . RUN apt-get update && apt install -y openssl RUN npm run postinstall --workspace=quadratic-api RUN npm run build:prod --workspace=quadratic-api diff --git a/quadratic-client/.dockerignore b/quadratic-client/.dockerignore deleted file mode 100644 index 5b5eab44ef..0000000000 --- a/quadratic-client/.dockerignore +++ /dev/null @@ -1,43 +0,0 @@ -# Environment files -.env* -**/.env* - -# Docker files -.dockerignore -**/.dockerignore -Dockerfile* -**/Dockerfile* -docker-compose* -**/docker-compose* - -# Build outputs -build/ -**/build/ -dist/ -**/dist/ -target/ -**/target/ - -# Dependencies -node_modules/ -**/node_modules/ -vendor/ -**/vendor/ - -# Test and coverage -coverage/ -**/coverage/ -*.test.* -**/*.test.* - -# Version control -.git/ -.gitignore -.gitattributes - -# IDE and editor files -.idea/ -.vscode/ -*.swp -*.swo -.DS_Store \ No newline at end of file diff --git a/quadratic-connection/.dockerignore b/quadratic-connection/.dockerignore deleted file mode 100644 index 5b5eab44ef..0000000000 --- a/quadratic-connection/.dockerignore +++ /dev/null @@ -1,43 +0,0 @@ -# Environment files -.env* -**/.env* - -# Docker files -.dockerignore -**/.dockerignore -Dockerfile* -**/Dockerfile* -docker-compose* -**/docker-compose* - -# Build outputs -build/ -**/build/ -dist/ -**/dist/ -target/ -**/target/ - -# Dependencies -node_modules/ -**/node_modules/ -vendor/ -**/vendor/ - -# Test and coverage -coverage/ -**/coverage/ -*.test.* -**/*.test.* - -# Version control -.git/ -.gitignore -.gitattributes - -# IDE and editor files -.idea/ -.vscode/ -*.swp -*.swo -.DS_Store \ No newline at end of file diff --git a/quadratic-files/.dockerignore b/quadratic-files/.dockerignore deleted file mode 100644 index 5b5eab44ef..0000000000 --- a/quadratic-files/.dockerignore +++ /dev/null @@ -1,43 +0,0 @@ -# Environment files -.env* -**/.env* - -# Docker files -.dockerignore -**/.dockerignore -Dockerfile* -**/Dockerfile* -docker-compose* -**/docker-compose* - -# Build outputs -build/ -**/build/ -dist/ -**/dist/ -target/ -**/target/ - -# Dependencies -node_modules/ -**/node_modules/ -vendor/ -**/vendor/ - -# Test and coverage -coverage/ -**/coverage/ -*.test.* -**/*.test.* - -# Version control -.git/ -.gitignore -.gitattributes - -# IDE and editor files -.idea/ -.vscode/ -*.swp -*.swo -.DS_Store \ No newline at end of file diff --git a/quadratic-multiplayer/.dockerignore b/quadratic-multiplayer/.dockerignore deleted file mode 100644 index 5b5eab44ef..0000000000 --- a/quadratic-multiplayer/.dockerignore +++ /dev/null @@ -1,43 +0,0 @@ -# Environment files -.env* -**/.env* - -# Docker files -.dockerignore -**/.dockerignore -Dockerfile* -**/Dockerfile* -docker-compose* -**/docker-compose* - -# Build outputs -build/ -**/build/ -dist/ -**/dist/ -target/ -**/target/ - -# Dependencies -node_modules/ -**/node_modules/ -vendor/ -**/vendor/ - -# Test and coverage -coverage/ -**/coverage/ -*.test.* -**/*.test.* - -# Version control -.git/ -.gitignore -.gitattributes - -# IDE and editor files -.idea/ -.vscode/ -*.swp -*.swo -.DS_Store \ No newline at end of file From 63d33b2b7f1f6c9a20374d0a0230df52c6626a33 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 08:32:34 +0530 Subject: [PATCH 34/49] seperate TS and rust layers --- .dockerignore | 7 ++++++- quadratic-client/Dockerfile | 18 +++++++++--------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/.dockerignore b/.dockerignore index f95554c258..271e5a1953 100644 --- a/.dockerignore +++ b/.dockerignore @@ -34,4 +34,9 @@ vendor/ .vscode/ *.swp *.swo -.DS_Store \ No newline at end of file +.DS_Store + +# TS/Rust types, this is generated during the build process +quadratic-client/src/app/quadratic-core/ +quadratic-client/src/app/quadratic-core-types/ +quadratic-client/src/app/quadratic-rust-client/ diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index 8c2dfd58d2..5b57170790 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -21,6 +21,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends python-is-pytho WORKDIR /quadratic +# Copy updateAlertVersion.json +COPY updateAlertVersion.json . + # Copy all package.json files COPY package.json . COPY package-lock.json . @@ -33,30 +36,27 @@ COPY ./quadratic-client/package*.json ./quadratic-client/ # Install npm dependencies RUN npm install --no-audit --no-fund -# Copy the rest of the application -COPY updateAlertVersion.json . -COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/ -COPY ./quadratic-core/. ./quadratic-core/ -COPY ./quadratic-rust-client/. ./quadratic-rust-client/ -COPY ./quadratic-shared/. ./quadratic-shared/ -COPY ./quadratic-client/. ./quadratic-client/ - # Run the packaging script for quadratic_py +COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/ RUN ./quadratic-kernels/python-wasm/package.sh --no-poetry # Build wasm and export TS/Rust types +COPY ./quadratic-core/. ./quadratic-core/ +COPY ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts ./quadratic-client/src/app/web-workers/quadraticCore/worker/rustCallbacks.ts RUN echo 'Building wasm...' && npm run build --workspace=quadratic-core # Build the quadratic-rust-client -# Layer caching will not work beyond this point on CI because of GIT_COMMIT being different for each commit +COPY ./quadratic-rust-client/. ./quadratic-rust-client/ ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client # Build the quadratic-shared +COPY ./quadratic-shared/. ./quadratic-shared/ RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared # Build the front-end +COPY ./quadratic-client/. ./quadratic-client/ ENV VITE_DEBUG=VITE_DEBUG_VAL ENV VITE_QUADRATIC_API_URL=VITE_QUADRATIC_API_URL_VAL ENV VITE_QUADRATIC_MULTIPLAYER_URL=VITE_QUADRATIC_MULTIPLAYER_URL_VAL From ebc70a56ff550c186528873862ff0aeb464dd137 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 08:33:03 +0530 Subject: [PATCH 35/49] use blacksmith build and push action --- .../preview-branches-publish-images.yml | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 931902091e..87012042a5 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -81,23 +81,24 @@ jobs: image=moby/buildkit:latest network=host - - name: Set up Docker layer cache - uses: useblacksmith/cache@v5 - with: - path: /tmp/.buildx-cache - key: buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} - restore-keys: | - buildx-${{ matrix.service }}- + # - name: Set up Docker layer cache + # uses: useblacksmith/cache@v5 + # with: + # path: /tmp/.buildx-cache + # key: buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} + # restore-keys: | + # buildx-${{ matrix.service }}- - name: Build and push - uses: docker/build-push-action@v6 + # uses: docker/build-push-action@v6 + uses: useblacksmith/build-push-action@v1 with: context: . file: quadratic-${{ matrix.service }}/Dockerfile push: true tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd + # cache-from: type=local,src=/tmp/.buildx-cache + # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} @@ -110,7 +111,7 @@ jobs: # Temp fix # https://github.com/docker/build-push-action/issues/252 # https://github.com/moby/buildkit/issues/1896 - - name: Update cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + # - name: Update cache + # run: | + # rm -rf /tmp/.buildx-cache + # mv /tmp/.buildx-cache-new /tmp/.buildx-cache From b1f4876fdecedbb45a742c0d65de4d08de81454f Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 09:04:27 +0530 Subject: [PATCH 36/49] prefer caching rust layers over npm dependencies --- .../preview-branches-publish-images.yml | 21 +------------------ quadratic-client/Dockerfile | 20 +++++++----------- 2 files changed, 9 insertions(+), 32 deletions(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 87012042a5..772001b137 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -58,7 +58,7 @@ jobs: run: | echo "REPO_NAME=quadratic-${{ matrix.service }}-development" >> $GITHUB_OUTPUT - - name: Create Private ECR Repository if not exists + - name: Create Private ECR Repository id: create-ecr env: REPO_NAME: ${{ steps.repo-name.outputs.REPO_NAME }} @@ -81,24 +81,13 @@ jobs: image=moby/buildkit:latest network=host - # - name: Set up Docker layer cache - # uses: useblacksmith/cache@v5 - # with: - # path: /tmp/.buildx-cache - # key: buildx-${{ matrix.service }}-pr-${{ github.event.pull_request.number }} - # restore-keys: | - # buildx-${{ matrix.service }}- - - name: Build and push - # uses: docker/build-push-action@v6 uses: useblacksmith/build-push-action@v1 with: context: . file: quadratic-${{ matrix.service }}/Dockerfile push: true tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} - # cache-from: type=local,src=/tmp/.buildx-cache - # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max,compression=zstd build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} @@ -107,11 +96,3 @@ jobs: labels: | org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }} org.opencontainers.image.revision=${{ github.sha }} - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - # - name: Update cache - # run: | - # rm -rf /tmp/.buildx-cache - # mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/quadratic-client/Dockerfile b/quadratic-client/Dockerfile index 5b57170790..1a4a0d2f3e 100644 --- a/quadratic-client/Dockerfile +++ b/quadratic-client/Dockerfile @@ -21,20 +21,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends python-is-pytho WORKDIR /quadratic -# Copy updateAlertVersion.json +# Copy root dependencies +# Any changes to these files will trigger a full rebuild (version change) COPY updateAlertVersion.json . - -# Copy all package.json files COPY package.json . -COPY package-lock.json . -COPY ./quadratic-kernels/python-wasm/package*.json ./quadratic-kernels/python-wasm/ -COPY ./quadratic-core/package*.json ./quadratic-core/ -COPY ./quadratic-rust-client/package*.json ./quadratic-rust-client/ -COPY ./quadratic-shared/package*.json ./quadratic-shared/ -COPY ./quadratic-client/package*.json ./quadratic-client/ - -# Install npm dependencies -RUN npm install --no-audit --no-fund # Run the packaging script for quadratic_py COPY ./quadratic-kernels/python-wasm/. ./quadratic-kernels/python-wasm/ @@ -51,6 +41,12 @@ ARG GIT_COMMIT ENV GIT_COMMIT=$GIT_COMMIT RUN echo 'Building quadratic-rust-client...' && npm run build --workspace=quadratic-rust-client +# Install npm dependencies +COPY package-lock.json . +COPY ./quadratic-shared/package*.json ./quadratic-shared/ +COPY ./quadratic-client/package*.json ./quadratic-client/ +RUN npm install --no-audit --no-fund + # Build the quadratic-shared COPY ./quadratic-shared/. ./quadratic-shared/ RUN echo 'Building quadratic-shared...' && npm run compile --workspace=quadratic-shared From b51c290dae1fffca52c0b0e75efdfb3af86889ba Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 09:15:40 +0530 Subject: [PATCH 37/49] add service name in tag --- .github/workflows/preview-branches-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 772001b137..26bc8f4f96 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -87,7 +87,7 @@ jobs: context: . file: quadratic-${{ matrix.service }}/Dockerfile push: true - tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} + tags: ${{ steps.create-ecr.outputs.ECR_URL }}:${{ matrix.service }}-pr-${{ github.event.pull_request.number }} build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} From 70af95bc7d86ae6064c7c00e945edc173fdaf03b Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 09:48:14 +0530 Subject: [PATCH 38/49] revert: add service name in tag, cache work without it --- .github/workflows/preview-branches-publish-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-branches-publish-images.yml index 26bc8f4f96..772001b137 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-branches-publish-images.yml @@ -87,7 +87,7 @@ jobs: context: . file: quadratic-${{ matrix.service }}/Dockerfile push: true - tags: ${{ steps.create-ecr.outputs.ECR_URL }}:${{ matrix.service }}-pr-${{ github.event.pull_request.number }} + tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} From 8b612aeea9aa864a93a9080716b5d1bee5cbc346 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 16:14:59 +0530 Subject: [PATCH 39/49] try cloudformation from github actions --- .../preview-cloudformation-deploy.yml | 77 ++++++ ...-images.yml => preview-publish-images.yml} | 3 +- .vscode/settings.json | 8 + infra/aws-cloudformation/preview.yml | 233 ++++++++++++++++++ 4 files changed, 320 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/preview-cloudformation-deploy.yml rename .github/workflows/{preview-branches-publish-images.yml => preview-publish-images.yml} (96%) create mode 100644 infra/aws-cloudformation/preview.yml diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml new file mode 100644 index 0000000000..37e0c0e0aa --- /dev/null +++ b/.github/workflows/preview-cloudformation-deploy.yml @@ -0,0 +1,77 @@ +name: Build Docker Images for Preview Branches + +on: + pull_request: + types: [opened, reopened] + +concurrency: + group: pr-${{ github.event.pull_request.number }} + +jobs: + deploy_cloudformation: + permissions: + contents: read + id-token: write + + runs-on: blacksmith-2vcpu-ubuntu-2204 + + env: + STACK_NAME: preview-pr-${{ github.event.pull_request.number }} + STACK_PARAMS: >- + ParameterKey=LicenseKey,ParameterValue=5a32bd8a-409e-4733-8846-1868c568a813 + ParameterKey=ImageTag,ParameterValue=pr-${{ github.event.pull_request.number }} + + steps: + - uses: actions/checkout@v4 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID_DEVELOPMENT }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY_DEVELOPMENT }} + aws-region: ${{ secrets.AWS_REGION }} + + - name: Deploy CloudFormation Stack + run: | + # Check if stack exists + if aws cloudformation describe-stacks --stack-name $STACK_NAME 2>/dev/null; then + # Update existing stack + aws cloudformation update-stack \ + --stack-name $STACK_NAME \ + --template-body file://infra/aws-cloudformation/preview.yml \ + --parameters $STACK_PARAMS \ + --capabilities CAPABILITY_IAM \ + --no-fail-on-empty-changeset + else + # Create new stack + aws cloudformation create-stack \ + --stack-name $STACK_NAME \ + --template-body file://infra/aws-cloudformation/preview.yml \ + --parameters $STACK_PARAMS \ + --capabilities CAPABILITY_IAM + + # Wait for stack creation to complete + aws cloudformation wait stack-create-complete --stack-name $STACK_NAME + fi + + - name: Get Stack Outputs + id: stack-output + run: | + DOMAIN=$(aws cloudformation describe-stacks \ + --stack-name $STACK_NAME \ + --query 'Stacks[0].Outputs[?OutputKey==`DomainRecord`].OutputValue' \ + --output text) + echo "PREVIEW_URL=https://$DOMAIN" >> $GITHUB_OUTPUT + + - name: Comment Preview URL + uses: actions/github-script@v7 + with: + script: | + const previewUrl = '${{ steps.stack-output.outputs.PREVIEW_URL }}'; + const message = `Preview URL: ${previewUrl}`; + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.name, + body: message + }); diff --git a/.github/workflows/preview-branches-publish-images.yml b/.github/workflows/preview-publish-images.yml similarity index 96% rename from .github/workflows/preview-branches-publish-images.yml rename to .github/workflows/preview-publish-images.yml index 772001b137..a07fac0fa0 100644 --- a/.github/workflows/preview-branches-publish-images.yml +++ b/.github/workflows/preview-publish-images.yml @@ -5,7 +5,7 @@ on: types: [opened, synchronize, reopened] concurrency: - group: ${{ github.head_ref }}-build-images + group: pr-${{ github.event.pull_request.number }} cancel-in-progress: true jobs: @@ -92,6 +92,7 @@ jobs: BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} GIT_SHA=${{ github.sha }} + BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }} PR_NUMBER=${{ github.event.pull_request.number }} labels: | org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }} diff --git a/.vscode/settings.json b/.vscode/settings.json index f0d8987b2b..9b548d8a97 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,19 +2,23 @@ "editor.formatOnSave": true, "cSpell.words": [ "actix", + "awscliv", "ayush", "bigdecimal", "bincode", "bindgen", "buildkit", "Buildx", + "containerd", "CRPXNLSKVLJFHH", "dashmap", "dbgjs", "dcell", "ddimaria", + "dearmor", "dgraph", "dotenv", + "dpkg", "endregion", "finitize", "Fuzzysort", @@ -31,6 +35,7 @@ "indicies", "itertools", "jwks", + "keyrings", "MDSL", "micropip", "minmax", @@ -51,6 +56,8 @@ "relcells", "reqwest", "scrollend", + "selfhost", + "selfhosted", "shadcn", "Signin", "smallpop", @@ -63,6 +70,7 @@ "undoable", "unspill", "useblacksmith", + "usermod", "vals", "vcpu", "websockets", diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml new file mode 100644 index 0000000000..c7195e4bb9 --- /dev/null +++ b/infra/aws-cloudformation/preview.yml @@ -0,0 +1,233 @@ +AWSTemplateFormatVersion: "2010-09-09" +Description: "Quadratic Selfhost Cloudformation Template" + +Parameters: + LicenseKey: + Type: String + Description: "Your license key for Quadratic. Get one here https://selfhost.quadratichq.com/" + ImageTag: + Type: String + Description: "Image tag to use for all services" + Default: "latest" + +Mappings: + EnvironmentConfig: + Preview: + DomainName: "quadratic-preview.com" + RegionMap: + us-west-2: + AMI: "ami-05134c8ef96964280" # Example AMI for us-west-2 (Ubuntu 20.04) + ap-south-1: + AMI: "ami-0522ab6e1ddcc7055" # Example AMI for ap-south-1 (Ubuntu 24.04 LTS) + +Resources: + EC2Role: + Type: AWS::IAM::Role + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: ec2.amazonaws.com + Action: sts:AssumeRole + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly + - arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore + Policies: + - PolicyName: ECRAccess + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - ecr:GetAuthorizationToken + - ecr:BatchCheckLayerAvailability + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:DescribeImages + - ecr:ListImages + Resource: "*" + + EC2InstanceProfile: + Type: AWS::IAM::InstanceProfile + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + Roles: + - !Ref EC2Role + + OpenSecurityGroup: + Type: "AWS::EC2::SecurityGroup" + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + GroupDescription: !Sub "${ImageTag}" + SecurityGroupIngress: + - IpProtocol: "tcp" + FromPort: "80" + ToPort: "80" + CidrIp: "0.0.0.0/0" + - IpProtocol: "tcp" + FromPort: "443" + ToPort: "443" + CidrIp: "0.0.0.0/0" + + EC2Instance: + Type: "AWS::EC2::Instance" + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + InstanceType: "m6a.large" + ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] + IamInstanceProfile: !Ref EC2InstanceProfile + SecurityGroups: + - !Ref OpenSecurityGroup + BlockDeviceMappings: + - DeviceName: "/dev/sda1" + Ebs: + VolumeSize: "25" + VolumeType: "gp3" + UserData: + Fn::Base64: !Sub | + #!/bin/bash + + # Update and install dependencies + sudo apt-get update + sudo apt-get install -y apt-transport-https ca-certificates curl software-properties-common unzip jq + + # Install AWS CLI v2 + curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" + unzip awscliv2.zip + sudo ./aws/install + rm -rf aws awscliv2.zip + + # Install Docker + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io + + # Install Docker Compose + sudo curl -L "https://github.com/docker/compose/releases/download/v2.21.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose + sudo chmod +x /usr/local/bin/docker-compose + + # Configure Docker + sudo systemctl enable docker + sudo systemctl start docker + sudo usermod -aG docker ubuntu + + # Export environment variables for docker-compose + echo 'export ECR_URL="${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com"' >> /home/ubuntu/.profile + echo 'export IMAGE_TAG="${ImageTag}"' >> /home/ubuntu/.profile + + # Source the new environment variables + source /home/ubuntu/.profile + + # Create login.sh script + echo '#!/bin/bash + aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com' > /quadratic-selfhost/login.sh + chmod +x /quadratic-selfhost/login.sh + + # Run Quadratic initialization + curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]} + + DnsRecords: + Type: AWS::Route53::RecordSet + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + HostedZoneName: !Sub "${!FindInMap [EnvironmentConfig, Preview, DomainName]}." + Name: !Sub "${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}" + Type: A + TTL: 300 + ResourceRecords: + - !GetAtt EC2Instance.PublicIp + + WildcardDnsRecord: + Type: AWS::Route53::RecordSet + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + HostedZoneName: !Sub "${!FindInMap [EnvironmentConfig, Preview, DomainName]}." + Name: !Sub "*.${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}" + Type: A + TTL: 300 + ResourceRecords: + - !GetAtt EC2Instance.PublicIp + + EventBridgeRole: + Type: AWS::IAM::Role + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: events.amazonaws.com + Action: sts:AssumeRole + Policies: + - PolicyName: InvokeSsmAutomation + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - ssm:SendCommand + Resource: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:*" + + EcrPushRule: + Type: AWS::Events::Rule + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + DependsOn: + - EC2Instance + - EventBridgeRole + Properties: + Description: "Rule to detect ECR image pushes" + EventPattern: + source: + - aws.ecr + detail-type: + - "ECR Image Action" + detail: + action-type: + - "PUSH" + repository-name: + - "quadratic-client-development" + - "quadratic-api-development" + - "quadratic-multiplayer-development" + - "quadratic-files-development" + - "quadratic-connection-development" + image-tag: + - !Ref ImageTag + State: "ENABLED" + Targets: + - Arn: !Sub "arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:automation-definition/AWS-RunShellScript" + Id: "TriggerScriptOnInstance" + RoleArn: !GetAtt EventBridgeRole.Arn + InputTransformer: + InputPathsMap: + "imageTag": "$.detail.image-tag" + "repository": "$.detail.repository-name" + InputTemplate: | + { + "InstanceIds": ["${EC2Instance.InstanceId}"], + "DocumentName": "AWS-RunShellScript", + "Parameters": { + "commands": [ + "cd /quadratic-selfhost", + "./login.sh", + "./start.sh", + ] + } + } + +Outputs: + DomainRecord: + Description: "Url of the selfhosted instance" + Value: !Sub "${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}" \ No newline at end of file From 0384d50a13cb8e25878a033ca710611c3cad1ba1 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 16:37:52 +0530 Subject: [PATCH 40/49] trigger CI --- .github/workflows/preview-cloudformation-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml index 37e0c0e0aa..3cbe2edf7c 100644 --- a/.github/workflows/preview-cloudformation-deploy.yml +++ b/.github/workflows/preview-cloudformation-deploy.yml @@ -1,4 +1,4 @@ -name: Build Docker Images for Preview Branches +name: Deploy CloudFormation Stack for Preview Branches on: pull_request: From deaa27586e8ee92fa44d91b88a2e32166c013428 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 16:41:37 +0530 Subject: [PATCH 41/49] concurrency group --- .github/workflows/preview-cloudformation-deploy.yml | 4 ++-- .github/workflows/preview-publish-images.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml index 3cbe2edf7c..064c6de0f3 100644 --- a/.github/workflows/preview-cloudformation-deploy.yml +++ b/.github/workflows/preview-cloudformation-deploy.yml @@ -1,11 +1,11 @@ -name: Deploy CloudFormation Stack for Preview Branches +name: Deploy CloudFormation Stack for Preview Branch on: pull_request: types: [opened, reopened] concurrency: - group: pr-${{ github.event.pull_request.number }} + group: pr-${{ github.event.pull_request.number }}-deploy-cloudformation jobs: deploy_cloudformation: diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml index a07fac0fa0..7b49f79b4c 100644 --- a/.github/workflows/preview-publish-images.yml +++ b/.github/workflows/preview-publish-images.yml @@ -5,7 +5,7 @@ on: types: [opened, synchronize, reopened] concurrency: - group: pr-${{ github.event.pull_request.number }} + group: pr-${{ github.event.pull_request.number }}-build-images cancel-in-progress: true jobs: From 0a995ccefbd7e910424566e2c17169245ea520c9 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 17:13:56 +0530 Subject: [PATCH 42/49] fix HostedZoneName --- .../workflows/preview-cloudformation-deploy.yml | 4 ++-- .github/workflows/preview-publish-images.yml | 4 ++-- infra/aws-cloudformation/preview.yml | 15 ++++++--------- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml index 064c6de0f3..bff4f28e2e 100644 --- a/.github/workflows/preview-cloudformation-deploy.yml +++ b/.github/workflows/preview-cloudformation-deploy.yml @@ -2,7 +2,7 @@ name: Deploy CloudFormation Stack for Preview Branch on: pull_request: - types: [opened, reopened] + types: [opened, synchronize, reopened] concurrency: group: pr-${{ github.event.pull_request.number }}-deploy-cloudformation @@ -16,7 +16,7 @@ jobs: runs-on: blacksmith-2vcpu-ubuntu-2204 env: - STACK_NAME: preview-pr-${{ github.event.pull_request.number }} + STACK_NAME: pr-${{ github.event.pull_request.number }} STACK_PARAMS: >- ParameterKey=LicenseKey,ParameterValue=5a32bd8a-409e-4733-8846-1868c568a813 ParameterKey=ImageTag,ParameterValue=pr-${{ github.event.pull_request.number }} diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml index 7b49f79b4c..4c4c099224 100644 --- a/.github/workflows/preview-publish-images.yml +++ b/.github/workflows/preview-publish-images.yml @@ -1,4 +1,4 @@ -name: Build Docker Images for Preview Branches +name: Build and Publish Images to ECR - Preview on: pull_request: @@ -9,7 +9,7 @@ concurrency: cancel-in-progress: true jobs: - build_images: + publish_images: permissions: contents: read diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml index c7195e4bb9..32e1b142e2 100644 --- a/infra/aws-cloudformation/preview.yml +++ b/infra/aws-cloudformation/preview.yml @@ -11,9 +11,6 @@ Parameters: Default: "latest" Mappings: - EnvironmentConfig: - Preview: - DomainName: "quadratic-preview.com" RegionMap: us-west-2: AMI: "ami-05134c8ef96964280" # Example AMI for us-west-2 (Ubuntu 20.04) @@ -132,15 +129,15 @@ Resources: chmod +x /quadratic-selfhost/login.sh # Run Quadratic initialization - curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]} + curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.quadratic-preview.com DnsRecords: Type: AWS::Route53::RecordSet DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: - HostedZoneName: !Sub "${!FindInMap [EnvironmentConfig, Preview, DomainName]}." - Name: !Sub "${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}" + HostedZoneName: "quadratic-preview.com." + Name: !Sub "${ImageTag}.quadratic-preview.com" Type: A TTL: 300 ResourceRecords: @@ -151,8 +148,8 @@ Resources: DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: - HostedZoneName: !Sub "${!FindInMap [EnvironmentConfig, Preview, DomainName]}." - Name: !Sub "*.${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}" + HostedZoneName: "quadratic-preview.com." + Name: !Sub "*.${ImageTag}.quadratic-preview.com" Type: A TTL: 300 ResourceRecords: @@ -230,4 +227,4 @@ Resources: Outputs: DomainRecord: Description: "Url of the selfhosted instance" - Value: !Sub "${ImageTag}.${!FindInMap [EnvironmentConfig, Preview, DomainName]}" \ No newline at end of file + Value: !Sub "${ImageTag}.quadratic-preview.com" From 55c770ba9dcad7559a72bacafc719b3a0e3b78ff Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Sun, 22 Dec 2024 17:20:17 +0530 Subject: [PATCH 43/49] fix InputTemplate --- infra/aws-cloudformation/preview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml index 32e1b142e2..74331749f9 100644 --- a/infra/aws-cloudformation/preview.yml +++ b/infra/aws-cloudformation/preview.yml @@ -219,7 +219,7 @@ Resources: "commands": [ "cd /quadratic-selfhost", "./login.sh", - "./start.sh", + "./start.sh" ] } } From 315520948da32f2aa31d9bcfd07e220c6f1e6e91 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Mon, 23 Dec 2024 02:33:28 +0530 Subject: [PATCH 44/49] fix login script --- .github/workflows/preview-cloudformation-deploy.yml | 2 +- infra/aws-cloudformation/preview.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml index bff4f28e2e..4f26dfc3ca 100644 --- a/.github/workflows/preview-cloudformation-deploy.yml +++ b/.github/workflows/preview-cloudformation-deploy.yml @@ -1,4 +1,4 @@ -name: Deploy CloudFormation Stack for Preview Branch +name: Deploy CloudFormation Stack - Preview on: pull_request: diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml index 74331749f9..8e5fb026ad 100644 --- a/infra/aws-cloudformation/preview.yml +++ b/infra/aws-cloudformation/preview.yml @@ -123,14 +123,14 @@ Resources: # Source the new environment variables source /home/ubuntu/.profile + # Run Quadratic initialization + curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.quadratic-preview.com + # Create login.sh script echo '#!/bin/bash aws ecr get-login-password --region ${AWS::Region} | docker login --username AWS --password-stdin ${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com' > /quadratic-selfhost/login.sh chmod +x /quadratic-selfhost/login.sh - # Run Quadratic initialization - curl -sSf https://raw.githubusercontent.com/quadratichq/quadratic-selfhost/main/init.sh -o init.sh && bash -i init.sh ${LicenseKey} ${ImageTag}.quadratic-preview.com - DnsRecords: Type: AWS::Route53::RecordSet DeletionPolicy: Delete From e17e26741b05e34fcfa1fd4e7f753b5c8b62c557 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Mon, 23 Dec 2024 03:27:39 +0530 Subject: [PATCH 45/49] fix github comment --- .../preview-cloudformation-deploy.yml | 4 ++-- .github/workflows/preview-publish-images.yml | 8 ++++---- infra/aws-cloudformation/preview.yml | 19 ++++++++++++++++++- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml index 4f26dfc3ca..3e4946b625 100644 --- a/.github/workflows/preview-cloudformation-deploy.yml +++ b/.github/workflows/preview-cloudformation-deploy.yml @@ -69,9 +69,9 @@ jobs: script: | const previewUrl = '${{ steps.stack-output.outputs.PREVIEW_URL }}'; const message = `Preview URL: ${previewUrl}`; - github.rest.issues.createComment({ - issue_number: context.issue.number, + await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.name, + issue_number: context.issue.number, body: message }); diff --git a/.github/workflows/preview-publish-images.yml b/.github/workflows/preview-publish-images.yml index 4c4c099224..8bbc2ddbf4 100644 --- a/.github/workflows/preview-publish-images.yml +++ b/.github/workflows/preview-publish-images.yml @@ -5,7 +5,7 @@ on: types: [opened, synchronize, reopened] concurrency: - group: pr-${{ github.event.pull_request.number }}-build-images + group: pr-${{ github.event.pull_request.number }}-publish-images cancel-in-progress: true jobs: @@ -87,13 +87,13 @@ jobs: context: . file: quadratic-${{ matrix.service }}/Dockerfile push: true - tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }} + tags: ${{ steps.create-ecr.outputs.ECR_URL }}:pr-${{ github.event.pull_request.number }}-${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} build-args: | BUILDKIT_INLINE_CACHE=1 BUILD_TIME=${{ steps.build-metadata.outputs.BUILD_TIME }} - GIT_SHA=${{ github.sha }} + GIT_SHA_SHORT=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} BRANCH_NAME=${{ steps.build-metadata.outputs.BRANCH_NAME }} PR_NUMBER=${{ github.event.pull_request.number }} labels: | org.opencontainers.image.created=${{ steps.build-metadata.outputs.BUILD_TIME }} - org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.revision=${{ steps.build-metadata.outputs.GIT_SHA_SHORT }} diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml index 8e5fb026ad..b6c50c94e9 100644 --- a/infra/aws-cloudformation/preview.yml +++ b/infra/aws-cloudformation/preview.yml @@ -8,7 +8,6 @@ Parameters: ImageTag: Type: String Description: "Image tag to use for all services" - Default: "latest" Mappings: RegionMap: @@ -23,6 +22,9 @@ Resources: DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: + Tags: + - Key: Name + Value: !Sub "pr-${ImageTag}" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: @@ -53,6 +55,9 @@ Resources: DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: + Tags: + - Key: Name + Value: !Sub "pr-${ImageTag}" Roles: - !Ref EC2Role @@ -61,6 +66,9 @@ Resources: DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: + Tags: + - Key: Name + Value: !Sub "pr-${ImageTag}" GroupDescription: !Sub "${ImageTag}" SecurityGroupIngress: - IpProtocol: "tcp" @@ -77,6 +85,9 @@ Resources: DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: + Tags: + - Key: Name + Value: !Sub "pr-${ImageTag}" InstanceType: "m6a.large" ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", AMI] IamInstanceProfile: !Ref EC2InstanceProfile @@ -160,6 +171,9 @@ Resources: DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: + Tags: + - Key: Name + Value: !Sub "pr-${ImageTag}" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: @@ -185,6 +199,9 @@ Resources: - EC2Instance - EventBridgeRole Properties: + Tags: + - Key: Name + Value: !Sub "pr-${ImageTag}" Description: "Rule to detect ECR image pushes" EventPattern: source: From ad15f68dfbdb63a99747fae2d747e54fdf192826 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Mon, 23 Dec 2024 03:30:54 +0530 Subject: [PATCH 46/49] remove tag on ec2 profile --- infra/aws-cloudformation/preview.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml index b6c50c94e9..a7a71962c1 100644 --- a/infra/aws-cloudformation/preview.yml +++ b/infra/aws-cloudformation/preview.yml @@ -55,9 +55,6 @@ Resources: DeletionPolicy: Delete UpdateReplacePolicy: Delete Properties: - Tags: - - Key: Name - Value: !Sub "pr-${ImageTag}" Roles: - !Ref EC2Role From 6555269fd2f89578463a717352667d7f17de654b Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Mon, 23 Dec 2024 03:56:11 +0530 Subject: [PATCH 47/49] use aws-cloudformation-github-deploy action --- .../preview-cloudformation-deploy.yml | 75 ++++++++----------- infra/aws-cloudformation/preview.yml | 13 +++- 2 files changed, 39 insertions(+), 49 deletions(-) diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml index 3e4946b625..579669ef47 100644 --- a/.github/workflows/preview-cloudformation-deploy.yml +++ b/.github/workflows/preview-cloudformation-deploy.yml @@ -1,4 +1,4 @@ -name: Deploy CloudFormation Stack - Preview +name: Deploy to AWS CloudFormation - Preview on: pull_request: @@ -17,9 +17,6 @@ jobs: env: STACK_NAME: pr-${{ github.event.pull_request.number }} - STACK_PARAMS: >- - ParameterKey=LicenseKey,ParameterValue=5a32bd8a-409e-4733-8846-1868c568a813 - ParameterKey=ImageTag,ParameterValue=pr-${{ github.event.pull_request.number }} steps: - uses: actions/checkout@v4 @@ -32,46 +29,34 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} - name: Deploy CloudFormation Stack - run: | - # Check if stack exists - if aws cloudformation describe-stacks --stack-name $STACK_NAME 2>/dev/null; then - # Update existing stack - aws cloudformation update-stack \ - --stack-name $STACK_NAME \ - --template-body file://infra/aws-cloudformation/preview.yml \ - --parameters $STACK_PARAMS \ - --capabilities CAPABILITY_IAM \ - --no-fail-on-empty-changeset - else - # Create new stack - aws cloudformation create-stack \ - --stack-name $STACK_NAME \ - --template-body file://infra/aws-cloudformation/preview.yml \ - --parameters $STACK_PARAMS \ - --capabilities CAPABILITY_IAM - - # Wait for stack creation to complete - aws cloudformation wait stack-create-complete --stack-name $STACK_NAME - fi - - - name: Get Stack Outputs - id: stack-output - run: | - DOMAIN=$(aws cloudformation describe-stacks \ - --stack-name $STACK_NAME \ - --query 'Stacks[0].Outputs[?OutputKey==`DomainRecord`].OutputValue' \ - --output text) - echo "PREVIEW_URL=https://$DOMAIN" >> $GITHUB_OUTPUT + id: deploy-stack + uses: aws-actions/aws-cloudformation-github-deploy@v1 + with: + name: ${{ env.STACK_NAME }} + template: infra/aws-cloudformation/preview.yml + parameter-overrides: >- + LicenseKey=5a32bd8a-409e-4733-8846-1868c568a813, + ImageTag=pr-${{ github.event.pull_request.number }} + capabilities: CAPABILITY_IAM + no-fail-on-empty-changeset: "1" + disable-rollback: false + + - name: Find Comment + uses: peter-evans/find-comment@v3 + id: preview-comment + with: + issue-number: ${{ github.event.pull_request.number }} + comment-author: 'github-actions[bot]' + body-includes: 'Preview Deployment' - - name: Comment Preview URL - uses: actions/github-script@v7 + - name: Create or update comment + if: steps.deploy-stack.outputs.DnsRecord != '' + uses: peter-evans/create-or-update-comment@v3 with: - script: | - const previewUrl = '${{ steps.stack-output.outputs.PREVIEW_URL }}'; - const message = `Preview URL: ${previewUrl}`; - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.name, - issue_number: context.issue.number, - body: message - }); + comment-id: ${{ steps.preview-comment.outputs.comment-id }} + issue-number: ${{ github.event.pull_request.number }} + body: | + ## Preview Deployment + Preview URL: https://${{ steps.deploy-stack.outputs.DnsRecord }} + edit-mode: replace + diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml index a7a71962c1..68211d8f38 100644 --- a/infra/aws-cloudformation/preview.yml +++ b/infra/aws-cloudformation/preview.yml @@ -16,6 +16,14 @@ Mappings: ap-south-1: AMI: "ami-0522ab6e1ddcc7055" # Example AMI for ap-south-1 (Ubuntu 24.04 LTS) +RollbackConfiguration: + MonitoringTimeInMinutes: 0 + RollbackTriggers: [] + +OnFailure: ROLLBACK +DeletionPolicy: Delete +UpdateReplacePolicy: Delete + Resources: EC2Role: Type: AWS::IAM::Role @@ -196,9 +204,6 @@ Resources: - EC2Instance - EventBridgeRole Properties: - Tags: - - Key: Name - Value: !Sub "pr-${ImageTag}" Description: "Rule to detect ECR image pushes" EventPattern: source: @@ -239,6 +244,6 @@ Resources: } Outputs: - DomainRecord: + DnsRecord: Description: "Url of the selfhosted instance" Value: !Sub "${ImageTag}.quadratic-preview.com" From 74ec7ce38178e60f3ee11e3f355548f61f6d0184 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Mon, 23 Dec 2024 04:03:45 +0530 Subject: [PATCH 48/49] remove rollback and delete attributes --- .github/workflows/preview-cloudformation-deploy.yml | 1 + infra/aws-cloudformation/preview.yml | 8 -------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml index 579669ef47..10912d404b 100644 --- a/.github/workflows/preview-cloudformation-deploy.yml +++ b/.github/workflows/preview-cloudformation-deploy.yml @@ -40,6 +40,7 @@ jobs: capabilities: CAPABILITY_IAM no-fail-on-empty-changeset: "1" disable-rollback: false + termination-protection: false - name: Find Comment uses: peter-evans/find-comment@v3 diff --git a/infra/aws-cloudformation/preview.yml b/infra/aws-cloudformation/preview.yml index 68211d8f38..23bc166c9a 100644 --- a/infra/aws-cloudformation/preview.yml +++ b/infra/aws-cloudformation/preview.yml @@ -16,14 +16,6 @@ Mappings: ap-south-1: AMI: "ami-0522ab6e1ddcc7055" # Example AMI for ap-south-1 (Ubuntu 24.04 LTS) -RollbackConfiguration: - MonitoringTimeInMinutes: 0 - RollbackTriggers: [] - -OnFailure: ROLLBACK -DeletionPolicy: Delete -UpdateReplacePolicy: Delete - Resources: EC2Role: Type: AWS::IAM::Role From 8bafe875c66a82d5a0d4638986b880f209e52c84 Mon Sep 17 00:00:00 2001 From: AyushAgrawal-A2 Date: Mon, 23 Dec 2024 04:14:15 +0530 Subject: [PATCH 49/49] add write permission for comment --- .github/workflows/preview-cloudformation-deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/preview-cloudformation-deploy.yml b/.github/workflows/preview-cloudformation-deploy.yml index 10912d404b..7c579d1514 100644 --- a/.github/workflows/preview-cloudformation-deploy.yml +++ b/.github/workflows/preview-cloudformation-deploy.yml @@ -12,6 +12,7 @@ jobs: permissions: contents: read id-token: write + pull-requests: write runs-on: blacksmith-2vcpu-ubuntu-2204