wip: upload-artifacts.sh, deploy and destroy scripts.

backend/scripts/.gitignore

@@ -1,2 +1,3 @@
 openapi3.yaml
 openapi2.yaml
+api_gateway_config.yaml

backend/scripts/convert_to_openapi2.py

@@ -37,6 +37,7 @@ def convert():
                 openapi3['paths'][path][method].pop('requestBody')
 
     openapi2['paths'].update(openapi3['paths'])
-
-    with open('openapi2.yaml', 'w') as f:
+
+    # this name should match the constant in iac/backend/prepare_backend.py
+    with open('api_gateway_config.yaml', 'w') as f:
         yaml.dump(openapi2, f)

iac/.gitignore

@@ -4,3 +4,4 @@ venv-iac
 realm/Pulumi.*.yaml
 environment/Pulumi.*.yaml
 keys/
+_tmp/

iac/README.md

@@ -106,7 +106,7 @@ Then, impersonate the service account that has the necessary roles to manage the
 ### Environment variables
 
 The deployment requires the following environment variables to be set:
-- `DEPLOYABLE_VERSION`: The deployable version of the application. This is used to tag the Docker images.
+- `ARTIFACTS_VERSION`: The artifacts version of the application. This is used to tag the Docker images.
 
 - `GCP_OAUTH_CLIENT_ID`: The OAuth client ID used to authenticate the application with Firebase.
 - `GCP_OAUTH_CLIENT_SECRET`: The OAuth client secret used to authenticate the application with Firebase.
@@ -137,7 +137,7 @@ content:
 
 ```shell
 # .env file
-DEPLOYABLE_VERSION=<DEPLOYABLE_VERSION>
+ARTIFACTS_VERSION=<ARTIFACTS_VERSION>
 
 GCP_OAUTH_CLIENT_ID=<GCP_OAUTH_CLIENT_ID>
 GCP_OAUTH_CLIENT_SECRET=<GCP_OAUTH_CLIENT_SECRET>

iac/backend/__main__.py

@@ -10,7 +10,7 @@
 
 from deploy_backend import deploy_backend, BackendServiceConfig
 
-from lib import getconfig, getstackref, getenv, parse_realm_env_name_from_stack, load_dot_realm_env
+from lib import getconfig, getstackref, getenv, parse_realm_env_name_from_stack, load_dot_realm_env, get_deployment_id
 
 
 def main():
@@ -56,7 +56,17 @@ def main():
         gcp_oauth_client_id=getenv("GCP_OAUTH_CLIENT_ID"),
     )
 
-    docker_image_tag = getenv("DEPLOYABLE_VERSION")
+    # version of the artifacts to deploy
+    artifacts_version = getenv("ARTIFACTS_VERSION")
+
+    # the key identifier of this deployment, used to identify the deployment.
+    run_number = getenv("DEPLOYMENT_RUN_NUMBER")
+
+    # deployment id will be used to know where to find the backend config,
+    # for now api_gateway_config
+    deployment_id = get_deployment_id(
+        run_number=run_number,
+        deploy_version=artifacts_version)
 
     # Deploy the backend
     deploy_backend(
@@ -66,7 +76,8 @@ def main():
         project_number=project_number,
         backend_service_cfg=backend_service_cfg,
         docker_repository=docker_repository,
-        docker_image_tag=docker_image_tag,
+        artifacts_version=artifacts_version,
+        deployment_id=deployment_id
     )
 
 

iac/backend/deploy_backend.py

@@ -1,11 +1,13 @@
 import base64
+import os.path
 from dataclasses import dataclass
 
 import pulumi
 import pulumi_gcp as gcp
 
 from pulumi import Output
 
+from backend.prepare_backend import CONFIGS_DIR, GCP_API_GATEWAY_CONFIG_FILE
 from lib.std_pulumi import ProjectBaseConfig, get_resource_name, get_project_base_config, get_file_as_string
 
 
@@ -32,8 +34,6 @@ class BackendServiceConfig:
     gcp_oauth_client_id: str
 
 
-GCP_API_GATEWAY_CONFIG_FILE = "./config/gcp_api_gateway_config.yaml"
-
 """
 # Set up GCP API Gateway.
 # The API Gateway will route the requests to the Compass Cloudrun instance. Additionally, it will verify the incoming JWT tokens
@@ -47,6 +47,7 @@ class BackendServiceConfig:
 def _setup_api_gateway(*,
                        basic_config: ProjectBaseConfig,
                        cloudrun: gcp.cloudrunv2.Service,
+                       deployment_id: str,
                        dependencies: list[pulumi.Resource]
                        ):
     apigw_service_account = gcp.serviceaccount.Account(
@@ -66,7 +67,8 @@ def _setup_api_gateway(*,
 
     # The GCP API Gateway uses OpenAPI 2.0 yaml files for the configurations.
     # The yaml must be base64 encoded.
-    apigw_config_yml_string = get_file_as_string(GCP_API_GATEWAY_CONFIG_FILE)
+    api_gateway_config_file_path: str = os.path.join(CONFIGS_DIR, deployment_id, GCP_API_GATEWAY_CONFIG_FILE).__str__()
+    apigw_config_yml_string = get_file_as_string(api_gateway_config_file_path)
 
     # update the yaml with the correct values
     # we are not using pulumi.Output.format because with path variables they are encapsulated in {}
@@ -90,8 +92,7 @@ def _setup_api_gateway(*,
         openapi_documents=[
             gcp.apigateway.ApiConfigOpenapiDocumentArgs(
                 document=gcp.apigateway.ApiConfigOpenapiDocumentDocumentArgs(
-                    # TODO: is the usage of path here correct?
-                    path=get_resource_name(resource="api-gateway", resource_type="config.yaml"),
+                    path=get_resource_name(resource="api-gateway", resource_type=GCP_API_GATEWAY_CONFIG_FILE),
                     contents=apigw_config_yaml_b64encoded,
                 ),
             )
@@ -258,7 +259,7 @@ def _deploy_cloud_run_service(
             ],
             service_account=service_account.email,
         ),
-        opts=pulumi.ResourceOptions(depends_on=dependencies+[iam_member], provider=basic_config.provider),
+        opts=pulumi.ResourceOptions(depends_on=dependencies + [iam_member], provider=basic_config.provider),
     )
     pulumi.export("cloud_run_url", service.uri)
     return service
@@ -273,7 +274,8 @@ def deploy_backend(
         project_number: Output[str],
         backend_service_cfg: BackendServiceConfig,
         docker_repository: pulumi.Output[gcp.artifactregistry.Repository],
-        docker_image_tag: str
+        artifacts_version: str,
+        deployment_id: str
 ):
     """
     Deploy the backend infrastructure
@@ -291,7 +293,7 @@ def deploy_backend(
     # get fully qualified image name
     fully_qualified_image_name = _get_fully_qualified_image_name(
         docker_repository=docker_repository,
-        tag=docker_image_tag
+        tag=artifacts_version
     )
 
     # Deploy the image as a cloud run service
@@ -305,5 +307,6 @@ def deploy_backend(
     _api_gateway = _setup_api_gateway(
         basic_config=basic_config,
         cloudrun=cloud_run,
+        deployment_id=deployment_id,
         dependencies=[cloud_run]
     )

iac/backend/prepare_backend.py

@@ -0,0 +1,55 @@
+import os
+import subprocess
+import sys
+
+# Determine the absolute path to the 'iac' directory
+iac_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
+# Add this directory to sys.path,
+# so that we can import the iac/lib module when we run pulumi from withing the iac/backend directory.
+sys.path.insert(0, iac_folder)
+
+from lib import get_pulumi_stack_outputs
+
+current_dir = os.path.join(iac_folder, "backend")
+
+# the GCP API config file name
+# it should match the file name at `backend/scripts/convert_to_openapi2.py`
+GCP_API_GATEWAY_CONFIG_FILE="api_gateway_config.yaml"
+CONFIGS_DIR = os.path.join(current_dir, "_tmp", "configs")
+
+
+def download_backend_config(*,
+                            realm_name: str,
+                            run_id: str,
+                            version: str) -> None:
+    output_dir = os.path.join(CONFIGS_DIR, run_id)
+    os.makedirs(output_dir, exist_ok=False)
+    print(f"Downloading the backend config to {output_dir}")
+
+    # download the configurations.
+    realm_outputs = get_pulumi_stack_outputs(stack_name=realm_name, module="realm")
+    generic_repository = realm_outputs["generic_repository"].value
+
+    try:
+        subprocess.run(
+            [
+                "gcloud",
+                "artifacts",
+                "generic",
+                "download",
+                "--package=backend-config",
+                f"--repository={generic_repository["name"]}",
+                f"--location={generic_repository["location"]}",
+                f"--project={generic_repository["project"]}",
+                "--destination=./",
+                f"--version={version}"
+            ],
+            cwd=output_dir,
+            check=True,
+            text=True
+        )
+
+        print("Done downloading the backend config bundle.")
+    except subprocess.CalledProcessError as e:
+        print(f"Error downloading backend config: {e}")
+        raise

iac/frontend/__main__.py

@@ -4,13 +4,13 @@
 # Determine the absolute path to the 'iac' directory
 libs_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 # Add this directory to sys.path,
-# so that we can import the iac/lib module when we run pulumi from withing the iac/frontend directory
+# so that we can import the iac/lib module when we run pulumi from withing the iac/frontend directory.
 sys.path.insert(0, libs_dir)
 
 import pulumi
 from deploy_frontend import deploy_frontend
 
-from lib.std_pulumi import getconfig, parse_realm_env_name_from_stack, getstackref, load_dot_realm_env
+from lib import getconfig, parse_realm_env_name_from_stack, getstackref, load_dot_realm_env, getenv, get_deployment_id
 
 
 def main():
@@ -27,8 +27,21 @@ def main():
     env_reference = pulumi.StackReference(f"tabiya-tech/compass-environment/{stack_name}")
     project = getstackref(env_reference, "project_id")
 
+    # the artifacts version of the frontend build to deploy.
+    artifacts_version = getenv("ARTIFACTS_VERSION")
+
+    # the unique identifier of this running deployment.
+    run_number = getenv("DEPLOYMENT_RUN_NUMBER")
+
+    # the deployment id, used to know, which source folder to get artifacts from.
+    deployment_id = get_deployment_id(run_number=run_number, deploy_version=artifacts_version, stack_name=stack_name)
+
     # Deploy the frontend
-    deploy_frontend(project, location)
+    deploy_frontend(
+        project=project,
+        location=location,
+        deployment_id=deployment_id
+    )
 
 
 if __name__ == "__main__":

iac/frontend/deploy_frontend.py

@@ -3,6 +3,7 @@
 import pulumi
 import pulumi_gcp as gcp
 
+from frontend.prepare_frontend import DEPLOYMENTS_DIR
 from lib.std_pulumi import ProjectBaseConfig, get_project_base_config, get_resource_name
 
 
@@ -55,18 +56,23 @@ def _make_bucket_public(basic_config: ProjectBaseConfig, bucket_name: pulumi.Out
     )
 
 
-def deploy_frontend(project: pulumi.Output[str], location: str):
+def deploy_frontend(*,
+                    project: pulumi.Output[str],
+                    location: str,
+                    deployment_id: str
+                    ):
     basic_config = get_project_base_config(project=project, location=location)
 
     bucket = _create_bucket(basic_config, "frontend")
 
-    new_ui_build_dir = "../../frontend-new/build"
-    _upload_directory_to_bucket(basic_config, bucket.name, new_ui_build_dir, "",
-                                ["index.html", "data/version.json"], [bucket])
-
-    frontend_out_dir = "../../frontend/out"
-    _upload_directory_to_bucket(basic_config, bucket.name, frontend_out_dir, "poc-ui",
-                                ["index.html"], [bucket])
+    frontend_artifacts_dir = os.path.join(DEPLOYMENTS_DIR, deployment_id)
+    _upload_directory_to_bucket(
+        basic_config,
+        bucket.name,
+        frontend_artifacts_dir,
+        "",
+        ["index.html", "data/version.json"],
+        [bucket])
 
     _make_bucket_public(basic_config, bucket.name, [bucket])