Migrate build scripts to python with single script to orchestrate during make up

Dockerfile

@@ -125,73 +125,15 @@ ENV DOCKER_TARGET=${DOCKER_TARGET}
 # Add our custom mime types (required for for ts/json/md files)
 COPY docker/etc/mime.types /etc/mime.types
 
-# Define production dependencies as a single layer
-# let's the rest of the stages inherit prod dependencies
-# and makes copying the /deps dir to the final layer easy.
-FROM base AS pip_production
-
-RUN \
-    --mount=type=bind,source=scripts/install_deps.py,target=${HOME}/scripts/install_deps.py \
-    # Files required to install pip dependencies
-    --mount=type=bind,source=./requirements/prod.txt,target=${HOME}/requirements/prod.txt \
-    # Files required to install npm dependencies
-    --mount=type=bind,source=package.json,target=/deps/package.json \
-    --mount=type=bind,source=package-lock.json,target=/deps/package-lock.json \
-    # Mounts for caching dependencies
-    --mount=type=cache,target=${PIP_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
-    --mount=type=cache,target=${NPM_CACHE_DIR},uid=${OLYMPIA_UID},gid=${OLYMPIA_UID} \
-<<EOF
-${HOME}/scripts/install_deps.py prod
-EOF
-
 FROM base AS development
 
-FROM base AS locales
-ARG LOCALE_DIR=${HOME}/locale
-# Compile locales
-# Copy the locale files from the host so it is writable by the olympia user
-COPY --chown=olympia:olympia locale ${LOCALE_DIR}
-# Copy the executable individually to improve the cache validity
-RUN \
-    --mount=type=bind,source=requirements/locale.txt,target=${HOME}/requirements/locale.txt \
-    --mount=type=bind,source=Makefile-docker,target=${HOME}/Makefile-docker \
-    --mount=type=bind,source=locale/compile-mo.sh,target=${HOME}/compile-mo.sh \
-    make -f Makefile-docker compile_locales
-
-# More efficient caching by mounting the exact files we need
-# and copying only the static/ & locale/ directory.
-FROM pip_production AS assets
-
-# In order to create js i18n files with all of our strings, we need to include
-# the compiled locale files
-COPY --from=locales --chown=olympia:olympia ${HOME}/locale/ ${HOME}/locale/
-# TODO: only copy the files we need for compiling assets
-COPY --chown=olympia:olympia static/ ${HOME}/static/
-
-# Finalize the build
-# TODO: We should move update_assets to the `builder` stage once we can efficiently
-# Run that command without having to copy the whole source code
-# This will shave nearly 1 minute off the best case build time
-RUN \
-    --mount=type=bind,src=src,target=${HOME}/src \
-    --mount=type=bind,src=Makefile-docker,target=${HOME}/Makefile-docker \
-    --mount=type=bind,src=manage.py,target=${HOME}/manage.py \
-<<EOF
-echo "from olympia.lib.settings_base import *" > settings_local.py
-DJANGO_SETTINGS_MODULE="settings_local" make -f Makefile-docker update_assets
-EOF
+# Copy build info from info
+COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
+
+FROM development AS production
 
-FROM base AS production
 # Copy the rest of the source files from the host
 COPY --chown=olympia:olympia . ${HOME}
-# Copy compiled locales from builder
-COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
-# Copy assets from assets
-COPY --from=assets --chown=olympia:olympia ${HOME}/site-static ${HOME}/site-static
-COPY --from=assets --chown=olympia:olympia ${HOME}/static-build ${HOME}/static-build
-# Copy build info from info
-COPY --from=info ${BUILD_INFO} ${BUILD_INFO}
-# Copy compiled locales from builder
-COPY --from=locales --chown=olympia:olympia ${HOME}/locale ${HOME}/locale
-# Copy dependencies from `pip_production`
-COPY --from=pip_production --chown=olympia:olympia /deps /deps
+
+# This command will install production dependencies and compile locales and static assets
+RUN ${HOME}/scripts/sync_host_files.py

Makefile-docker

@@ -72,14 +72,8 @@ data_load:
 	./manage.py data_load $(ARGS)
 
 .PHONY: update_assets
-update_assets:
-	# Copy files required in compress_assets to the static folder
-	# If changing this here, make sure to adapt tests in amo/test_commands.py
-	$(PYTHON_COMMAND) manage.py compress_assets
-	$(PYTHON_COMMAND) manage.py generate_jsi18n_files
-	# Collect static files: This MUST be run last or files will be missing
-	$(PYTHON_COMMAND) manage.py collectstatic --noinput
-
+update_assets: ## Update the static assets
+	$(HOME)/scripts/update_assets.py
 
 .PHONY: update_deps
 update_deps: ## Update the dependencies
@@ -218,7 +212,7 @@ extract_locales: ## extracts and merges translation strings
 .PHONE: compile_locales
 compile_locales: ## compiles translation strings
 	$(PIP_COMMAND) install --progress-bar=off --no-deps -r requirements/locale.txt
-	./locale/compile-mo.sh ./locale/
+	$(HOME)/scripts/compile_locales.py
 
 .PHONY: help_submake
 help_submake:

Makefile-os

@@ -153,17 +153,17 @@ docker_clean_build_cache: ## Remove buildx build cache
 .PHONY: clean_docker
 clean_docker: docker_compose_down docker_mysqld_volume_remove docker_clean_images docker_clean_volumes docker_clean_build_cache ## Remove all docker resources taking space on the host machine
 
-.PHONY: docker_update_deps
-docker_update_deps: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
+.PHONY: docker_sync_host
+docker_sync_host: docker_mysqld_volume_create ## Update the dependencies in the container based on the docker tag and target
 	docker compose run \
 		--rm \
 		--no-deps \
 		$(DOCKER_RUN_ARGS) \
 		web \
-		make update_deps
+		./scripts/sync_host_files.py
 
 .PHONY: up_pre
-up_pre: setup docker_pull_or_build docker_update_deps ## Pre-up the environment, setup files, volumes and host state
+up_pre: setup docker_pull_or_build docker_sync_host ## Pre-up the environment, setup files, volumes and host state
 
 .PHONY: up_start
 up_start: docker_mysqld_volume_create ## Start the docker containers

scripts/compile_locales.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+
+import os
+import subprocess
+from concurrent.futures import ThreadPoolExecutor
+
+
+def process_po_file(pofile, attempt=0):
+    """Process a single .po file, creating corresponding .mo file."""
+    print('processing', pofile)
+    directory = os.path.dirname(pofile)
+    stem = os.path.splitext(os.path.basename(pofile))[0]
+    mo_path = os.path.join(directory, f'{stem}.mo')
+
+    # Touch the .mo file
+    open(mo_path, 'a').close()
+
+    try:
+        # Run dennis-cmd lint
+        subprocess.run(
+            ['dennis-cmd', 'lint', '--errorsonly', pofile],
+            capture_output=True,
+            check=False,
+        )
+        # If lint passes, run msgfmt
+        subprocess.run(['msgfmt', '-o', mo_path, pofile], check=True)
+        return
+    except subprocess.CalledProcessError as e:
+        if attempt < 3:
+            print(f'Failed attempt {attempt} for {pofile}, retrying...')
+            return process_po_file(pofile, attempt=attempt + 1)
+        raise e
+
+
+def main():
+    # Ensure 'dennis' is installed
+    try:
+        import dennis as _
+    except ImportError:
+        print(
+            'Error: dennis is not installed. Please install it with pip install dennis'
+        )
+        exit(1)
+
+    locale_dir = os.path.abspath(
+        os.path.join(
+            os.path.dirname(__file__),
+            '..',
+            'locale',
+        )
+    )
+
+    print(f'Compiling locales in {locale_dir}')
+
+    # Collect all files first
+    django_files = []
+    djangojs_files = []
+    for root, _, files in os.walk(locale_dir):
+        for file in files:
+            if file == 'django.po':
+                django_files.append(os.path.join(root, file))
+            elif file == 'djangojs.po':
+                djangojs_files.append(os.path.join(root, file))
+
+    # Process django.po files in parallel
+    with ThreadPoolExecutor() as executor:
+        executor.map(process_po_file, django_files + djangojs_files)
+
+
+if __name__ == '__main__':
+    main()

scripts/sync_host_files.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+
+import json
+import os
+import subprocess
+
+
+def main():
+    BUILD_INFO = os.environ.get('BUILD_INFO')
+
+    subprocess.run(['make', 'update_deps'], check=True)
+
+    with open(BUILD_INFO, 'r') as f:
+        build_info = json.load(f)
+
+    if build_info.get('target') == 'production':
+        subprocess.run(['make', 'compile_locales'], check=True)
+        subprocess.run(['make', 'update_assets'], check=True)
+
+
+if __name__ == '__main__':
+    main()

scripts/update_assets.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import os
+import shutil
+import subprocess
+
+
+def main():
+    HOME = os.environ.get('HOME')
+    STATIC_DIRS = ['static-build', 'site-static']
+
+    for dir in STATIC_DIRS:
+        path = os.path.join(HOME, dir)
+        os.makedirs(path, exist_ok=True)
+        for file in os.listdir(path):
+            file_path = os.path.join(path, file)
+            print(f'Removing {file_path}')
+            if os.path.isdir(file_path):
+                shutil.rmtree(file_path)
+            else:
+                os.remove(file_path)
+
+    script_prefix = ['python3', 'manage.py']
+
+    environment = os.environ.copy()
+    # Always run in production mode without any development settings
+    environment['DJANGO_SETTINGS_MODULE'] = 'olympia.lib.settings_base'
+
+    subprocess.run(
+        script_prefix + ['compress_assets'],
+        check=True,
+        env=environment,
+    )
+    subprocess.run(
+        script_prefix + ['generate_jsi18n_files'],
+        check=True,
+        env=environment,
+    )
+    subprocess.run(
+        script_prefix + ['collectstatic', '--noinput'],
+        check=True,
+        env=environment,
+    )
+
+
+if __name__ == '__main__':
+    main()

src/olympia/lib/settings_base.py

@@ -1331,7 +1331,7 @@ def read_only_mode(env):
 NODE_PACKAGE_JSON = os.path.join('/', 'deps', 'package.json')
 NODE_PACKAGE_MANAGER_INSTALL_OPTIONS = ['--dry-run']
 
-STATIC_BUILD_PATH = os.path.join('/', 'data', 'olympia', 'static-build')
+STATIC_BUILD_PATH = path('static-build')
 
 STATICFILES_DIRS = (
     path('static'),