Reduce-docker-buildtime

.dockerignore

@@ -1,3 +1,17 @@
 .dockerignore
+Dockerfile
+Dockerfile.*
+docker-compose.*
 node_modules
-docker-compose.yaml
+docker-compose.yaml
+similarity_api/
+docs/
+bin/
+deployment_config/
+terraform/
+**/node_modules
+**/.next
+**/dist
+*.md
+reports
+.env

.env.example

@@ -27,6 +27,7 @@ REDIRECT_URI_HOST=http://localhost:8080
 CURRENT_USER_ID=1
 # NEW_RELIC_LICENSE_KEY can be omitted in local development
 NEW_RELIC_LICENSE_KEY=secret_key
+NEW_RELIC_ENABLED=false 
 # Set to false to require user to go through auth flow, never true in production envs
 BYPASS_AUTH=true
 HSES_DATA_FILE_URL=url

.gitignore

@@ -14,7 +14,8 @@ tests/e2e/report
 /doc
 
 # Build related
-build/
+build/**
+dist/**
 frontend/build
 node_modules
 

Dockerfile.dev

@@ -0,0 +1,59 @@
+FROM node:18.18.2 as base
+WORKDIR /app
+RUN apt-get update && apt-get install -y --no-install-recommends dumb-init lcov chromium
+ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true \
+ PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
+COPY package.json yarn.lock ./
+RUN yarn global add node-gyp &&\
+ yarn install &&\ 
+ yarn cache clean
+
+FROM node:18.18.2-alpine as frontend-base
+WORKDIR /app
+RUN apk update && apk add yarn dumb-init
+COPY frontend/package.json frontend/yarn.lock ./
+RUN yarn install &&\ 
+ yarn cache clean
+
+FROM node:18.18.2 as build
+ENV NODE_OPTIONS=--max_old_space_size=2048
+WORKDIR /app
+COPY --from=base /app/node_modules /app/node_modules
+COPY --from=base /app/package.json /app/package.json
+COPY src/ /app/src/
+COPY tests/ /app/tests/
+COPY config/ /app/config/
+RUN yarn tsc --build --verbose src
+
+FROM node:18.18.2-alpine as frontend-build
+ENV NODE_OPTIONS=--max_old_space_size=2048
+WORKDIR /app
+ENV DISABLE_ESLINT_PLUGIN=true
+COPY --from=frontend-base /app/node_modules /app/node_modules
+COPY ./frontend .
+RUN yarn build:local
+
+FROM node:18.18.2-alpine as frontend
+RUN apk update
+# ENV CHOKIDAR_USEPOLLING=true
+WORKDIR /app
+COPY --chown=node:node --from=frontend-base /usr/bin/dumb-init /usr/bin/dumb-init
+COPY --chown=node:node --from=frontend-build /app/build /app
+COPY --chown=node:node --from=frontend-build /app/node_modules /app/node_modules
+COPY --chown=node:node --from=frontend-build /app/package.json /app/package.json
+USER node
+# CMD ["/usr/bin/dumb-init", "node", "server.js"]
+
+FROM node:18.18.2-slim as backend
+WORKDIR /app
+COPY --chown=node:node .sequelizerc /app/.sequelizerc
+COPY --chown=node:node --from=base /usr/bin/dumb-init /usr/bin/dumb-init
+COPY --chown=node:node --from=build /app/build /app
+COPY --chown=node:node --from=build /app/node_modules/ /app/node_modules/
+COPY --chown=node:node --from=build /app/package.json /app/package.json
+USER node
+# CMD ["dumb-init", "node", "server.js"]
+
+FROM backend as worker
+
+FROM backend as testingonly

docker-compose.override.yml

@@ -3,18 +3,26 @@ services:
  backend:
  build:
  context: .
+ dockerfile: ./Dockerfile.dev
+ cache_from: 
+ - node:18.18.2 # Using the base image as a cache source
+ - node:18.18.2-slim
+ - node:18.18.2-alpine
+ - head-start-ttadp-backend:latest
+ target: backend
  command: yarn server
  user: ${CURRENT_USER:-root}
  ports:
  - "8080:8080"
  depends_on:
  - db
  - redis
- environment:
- - POSTGRES_HOST=postgres_docker
- - REDIS_HOST=redis
- - SMTP_HOST=mailcatcher
- - FONTAWESOME_NPM_AUTH_TOKEN
+ env_file: .env
+ # environment:
+ # POSTGRES_HOST: postgres_docker
+ # REDIS_HOST: redis
+ # SMTP_HOST: mailcatcher
+ # FONTAWESOME_NPM_AUTH_TOKEN:
  # On an M1 mac, puppeteer install fails with the message:
  # "The chromium binary is not available for arm64"
  #
@@ -25,13 +33,34 @@ services:
  # In ~/.zshrc (in my case) add:
  # export PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
  # export PUPPETEER_EXECUTABLE_PATH=`which chromium`
- - PUPPETEER_EXECUTABLE_PATH
- - PUPPETEER_SKIP_CHROMIUM_DOWNLOAD
+ # PUPPETEER_EXECUTABLE_PATH:
+ # PUPPETEER_SKIP_CHROMIUM_DOWNLOAD:
+ # POSTGRES_DB: ${POSTGRES_DB}
+ # POSTGRES_USER: ${POSTGRES_USERNAME}
+ # POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+ # BACKEND_PROXY: ${BACKEND_PROXY}
+ # AUTH_CLIENT_SECRET: ${AUTH_CLIENT_SECRET}
+ # SESSION_SECRET: ${SESSION_SECRET}
+ # JWT_SECRET: ${JWT_SECRET}
+ # REDIRECT_URI_HOST: ${REDIRECT_URI_HOST}
+ # AUTH_BASE: ${AUTH_BASE}
  volumes:
- - ".:/app:rw"
+ - "./src/:/app/src/:rw"
+ - "./tests/:/app/tests/:rw"
+ - "./config/:/app/config:rw"
+ - "./packages:/packages:ro"
+ - ".env:/app/.env:ro"
  frontend:
  build:
  context: .
+ dockerfile: ./Dockerfile.dev
+ cache_from: 
+ - node:18.18.2 # Using the base image as a cache source
+ - node:18.18.2-slim
+ - node:18.18.2-alpine
+ - head-start-ttadp-frontend:latest
+ target: frontend
+ env_file: .env
  command: yarn start
  user: ${CURRENT_USER:-root}
  stdin_open: true
@@ -41,24 +70,38 @@ services:
  - "./frontend:/app:rw"
  - "./scripts:/app/scripts"
  - "./packages:/packages:ro"
- environment:
- - BACKEND_PROXY=http://backend:8080
- - FONTAWESOME_NPM_AUTH_TOKEN
- - REACT_APP_WEBSOCKET_URL
+ - ".env:/app/.env:ro"
+ # environment:
+ # REDIRECT_URI_HOST: ${REDIRECT_URI_HOST}
+ # CURRENT_USER_ID: ${CURRENT_USER_ID}
+ # BYPASS_AUTH: ${BYPASS_AUTH}
+ # FONTAWESOME_NPM_AUTH_TOKEN: ${FONTAWESOME_NPM_AUTH_TOKEN}
+ # REACT_APP_WEBSOCKET_URL: REACT_APP_WEBSOCKET_URL
+ # BACKEND_PROXY: ${BACKEND_PROXY}
  worker:
  build:
  context: .
+ dockerfile: ./Dockerfile.dev
+ cache_from: 
+ - node:18.18.2 # Using the base image as a cache source
+ - node:18.18.2-slim
+ - node:18.18.2-alpine
+ - head-start-ttadp-worker:latest
+ target: backend
  command: yarn worker
  env_file: .env
  depends_on:
  - db
  - redis
- environment:
- - POSTGRES_HOST=postgres_docker
- - REDIS_HOST=redis
- - SMTP_HOST=mailcatcher
+ # environment:
+ #  - POSTGRES_HOST=postgres_docker
+ #  - REDIS_HOST=redis
+ #  - SMTP_HOST=mailcatcher
  volumes:
- - ".:/app:rw"
+ - "./src/:/app/src/:rw"
+ - "./tests/:/app/tests/:rw"
+ - "./config/:/app/config:rw"
+ - ".env:/app/.env:ro"
  owasp_zap_backend:
  image: owasp/zap2docker-stable:latest
  platform: linux/arm64
@@ -73,7 +116,7 @@ services:
  image: owasp/zap2docker-stable:latest
  platform: linux/arm64
  user: zap
- command: zap-api-scan.py -t http://similarity:8080/openapi.json -f openapi -I -i -r owasp_api_report.html
+ command: zap-api-scan.py -t http://similarity_api:8080/openapi.json -f openapi -I -i -r owasp_api_report.html
  volumes:
  - ./zap.conf:/zap/wrk/zap.conf:ro
  - ./reports:/zap/wrk:rw

docker-compose.yml

@@ -17,6 +17,8 @@ services:
  volumes:
  - dbdata:/var/lib/postgresql/data
  shm_size: 1g
+ environment:
+ POSTGRES_HOST: localhost 
  minio:
  image: minio/minio:RELEASE.2024-01-01T16-36-33Z
  env_file: .env
@@ -33,7 +35,7 @@ services:
  depends_on:
  - minio
  clamav-rest:
- image: ajilaag/clamav-rest
+ image: kcirtapfromspace/clamav-rest
  ports:
  - "9443:9443"
  environment:
@@ -50,6 +52,10 @@ services:
  - "./similarity_api/src:/app:rw"
  redis:
  image: redis:5.0.6-alpine
+ # https://github.com/docker-library/redis/issues/191
+ sysctls:
+ - net.core.somaxconn=511
+ # - vm.overcommit_memory=1
  command: ['redis-server', '--requirepass', '$REDIS_PASS']
  env_file: .env
  ports:
@@ -76,12 +82,20 @@ services:
  testingonly:
  build:
  context: .
+ dockerfile: ./Dockerfile.dev
+ # target: dependencies
+ cache_from: 
+ - node:18.18.2 # Using the base image as a cache source
+ - head-start-ttadp-testingonly:latest
+ target: backend
  ports:
  - "9999:9999"
  depends_on:
  - db
  volumes:
- - ".:/app:rw"
+ - "./src/:/app/src/:rw"
+ - "./tests/:/app/tests/:rw"
+ - "./config/:/app/config:rw"
  command: yarn start:testingonly
  environment:
  - POSTGRES_HOST=postgres_docker

frontend/package.json

@@ -88,7 +88,6 @@
  "tar": "^4.4.18",
  "d3-color": "^3.1.0",
  "minimatch": "^3.0.5",
- "postcss": "^8.4.31",
  "decode-uri-component": "^0.2.1",
  "tough-cookie": "^4.0.0",
  "ua-parser-js": "^0.7.33",
@@ -159,6 +158,7 @@
  ]
  },
  "devDependencies": {
+ "postcss": "^8.4.33",
  "@sheerun/mutationobserver-shim": "^0.3.3",
  "@testing-library/dom": "^8.11.1",
  "@testing-library/jest-dom": "^5.11.9",
@@ -183,6 +183,9 @@
  "react-scripts": "^5.0.1",
  "react-select-event": "^5.1.0"
  },
+ "peerDependencies": {
+ "postcss": "^8.4.33"
+ },
  "jest": {
  "coveragePathIgnorePatterns": [
  "<rootDir>/src/index.js",

frontend/yarn.lock

@@ -3799,9 +3799,9 @@ caniuse-api@^3.0.0:
  lodash.uniq "^4.5.0"
 
 caniuse-lite@^1.0.0, caniuse-lite@^1.0.30001400, caniuse-lite@^1.0.30001426:
- version "1.0.30001441"
- resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001441.tgz#987437b266260b640a23cd18fbddb509d7f69f3e"
- integrity sha512-OyxRR4Vof59I3yGWXws6i908EtGbMzVUi3ganaZQHmydk1iwDhRnvaPG2WaR0KcqrDFKrxVZHULT396LEPhXfg==
+ version "1.0.30001583"
+ resolved "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001583.tgz"
+ integrity sha512-acWTYaha8xfhA/Du/z4sNZjHUWjkiuoAi2LM+T/aL+kemKQgPT1xBb/YKjlQ0Qo8gvbHsGNplrEJ+9G3gL7i4Q==
 
 canvas-fit@^1.5.0:
  version "1.5.0"

package.json

@@ -15,8 +15,8 @@
  "start:web": "node ./build/server/src/index.js",
  "start:worker": "node ./build/server/src/worker.js",
  "start:testingonly": "tsx watch ./src/testingOnly.js",
- "server": "tsx watch src/index.ts",
- "worker": "tsx watch src/worker.ts",
+ "server": "tsx watch --trace-warnings src/index.ts",
+ "worker": "tsx watch --trace-warnings src/worker.ts",
  "server:debug": "nodemon --inspect=0.0.0.0:9229 src/index.js --exec babel-node",
  "client": "yarn --cwd frontend start",
  "test": "yarn build && jest build/server/src --runInBand",
@@ -52,28 +52,28 @@
  "db:seed:prod": "node_modules/.bin/sequelize db:seed:all --options-path .production.sequelizerc",
  "db:seed:undo": "node_modules/.bin/sequelize db:seed:undo:all",
  "db:seed:undo:prod": "node_modules/.bin/sequelize db:seed:undo:all --options-path .production.sequelizerc",
- "docker:deps": "docker-compose run --rm backend yarn global add node-gyp && docker-compose run --rm backend yarn install && docker-compose run --rm frontend yarn global add node-gyp && docker-compose run --rm frontend yarn install && docker-compose run --rm testingonly yarn global add node-gyp && docker-compose run --rm testingonly yarn install",
+ "docker:deps": "docker compose run --rm backend yarn global add node-gyp && docker compose run --rm backend yarn install && docker compose run --rm frontend yarn global add node-gyp && docker compose run --rm frontend yarn install && docker compose run --rm testingonly yarn global add node-gyp && docker compose run --rm testingonly yarn install",
  "docker:reset": "./bin/reset-all",
- "docker:start": "docker-compose up",
- "docker:start:debug": "docker-compose --compatibility -f docker-compose.yml -f docker-compose.debug.yml up",
- "docker:stop": "docker-compose down",
- "docker:dbs:start": "docker-compose -f 'docker-compose.yml' up",
- "docker:dbs:stop": "docker-compose -f 'docker-compose.yml' down",
+ "docker:start": "docker compose up",
+ "docker:start:debug": "docker compose --compatibility -f docker compose.yml -f docker compose.debug.yml up",
+ "docker:stop": "docker compose down",
+ "docker:dbs:start": "docker compose -f 'docker compose.yml' up",
+ "docker:dbs:stop": "docker compose -f 'docker compose.yml' down",
  "docker:test": "./bin/run-tests",
- "docker:test:be": "docker-compose run --rm backend yarn test",
- "docker:lint": "docker-compose run --rm backend yarn lint:ci && docker-compose run --rm frontend yarn lint:ci",
- "docker:lint:fix": "docker-compose run --rm backend yarn lint:fix && docker-compose run --rm frontend yarn lint:fix",
- "docker:shell:frontend": "docker-compose run --rm frontend /bin/bash",
- "docker:shell:backend": "docker-compose run --rm backend /bin/bash",
- "docker:db:migrate": "docker-compose run --rm backend node_modules/.bin/sequelize db:migrate && yarn docker:ldm",
- "docker:db:migrate:undo": "docker-compose run --rm backend node_modules/.bin/sequelize db:migrate:undo",
- "docker:db:seed": "docker-compose run --rm backend yarn db:seed",
- "docker:db:seed:undo": "docker-compose run --rm backend yarn db:seed:undo",
- "docker:ldm": "docker-compose run --rm backend yarn ldm",
- "docker:makecolors": "docker-compose run --rm frontend yarn makecolors",
- "docker:yarn": "docker-compose run yarn",
- "docker:yarn:fe": "docker-compose run --rm frontend yarn",
- "docker:yarn:be": "docker-compose run --rm backend yarn",
+ "docker:test:be": "docker compose run --rm backend yarn test",
+ "docker:lint": "docker compose run --rm backend yarn lint:ci && docker compose run --rm frontend yarn lint:ci",
+ "docker:lint:fix": "docker compose run --rm backend yarn lint:fix && docker compose run --rm frontend yarn lint:fix",
+ "docker:shell:frontend": "docker compose run --rm frontend /bin/bash",
+ "docker:shell:backend": "docker compose run --rm backend /bin/bash",
+ "docker:db:migrate": "docker compose run --rm backend node_modules/.bin/sequelize db:migrate && yarn docker:ldm",
+ "docker:db:migrate:undo": "docker compose run --rm backend node_modules/.bin/sequelize db:migrate:undo",
+ "docker:db:seed": "docker compose run --rm backend yarn db:seed",
+ "docker:db:seed:undo": "docker compose run --rm backend yarn db:seed:undo",
+ "docker:ldm": "docker compose run --rm backend yarn ldm",
+ "docker:makecolors": "docker compose run --rm frontend yarn makecolors",
+ "docker:yarn": "docker compose run yarn",
+ "docker:yarn:fe": "docker compose run --rm frontend yarn",
+ "docker:yarn:be": "docker compose run --rm backend yarn",
  "import:reports:local": "./node_modules/.bin/babel-node ./src/tools/importSSActivityReports.js",
  "import:reports": "node ./build/server/src/tools/importSSActivityReports.js",
  "import:goals:local": "./node_modules/.bin/babel-node ./src/tools/importTTAPlanGoals.js",
@@ -245,6 +245,7 @@
  ]
  },
  "devDependencies": {
+ "turbo": "^1.9.6",
  "@axe-core/cli": "4.6.0",
  "@axe-core/playwright": "^4.6.0",
  "@babel/cli": "^7.11.6",

tiltfile

@@ -0,0 +1 @@
+docker_compose(["./docker-compose.yml", "./docker-compose.override.yml"])