From e4890a83bb9297d7b599048100f106030bcbc5fd Mon Sep 17 00:00:00 2001 From: Daniel Pierce Date: Tue, 19 Sep 2023 17:52:37 -0400 Subject: [PATCH] dassie/koppie alignment and added bundle caching (#6325) * dassie/koppie alignment and added bundle caching - Use the same images for dassie/koppie dependent services. - Don't build an image for db_migrate, reuse the app image. - Use `dev-entrypoint.sh` to copy gems from image build to `/app/bundle` volume to speed up runtime `bundle install`. Only set in compose to avoid affecting CircleCI build. - `/bin` is volume mounted to allow easier dev of entrypoints, etc. --------- Co-authored-by: Alisha Evans --- .dassie/.env | 4 +++- .dassie/config/redis.yml | 2 ++ .koppie/.env | 3 ++- .koppie/config/application.rb | 2 +- bin/dev-entrypoint.sh | 13 +++++++++++++ docker-compose-koppie.yml | 29 ++++++++++++++++++----------- docker-compose.yml | 34 ++++++++++++++++++++++------------ 7 files changed, 61 insertions(+), 26 deletions(-) create mode 100755 bin/dev-entrypoint.sh diff --git a/.dassie/.env b/.dassie/.env index 9f13862102..077bfe8a08 100644 --- a/.dassie/.env +++ b/.dassie/.env @@ -1,5 +1,6 @@ ANALYTICS_START_DATE=2021-08-21 BUNDLE_GEMFILE=Gemfile.dassie +BUNDLE_PATH=/app/bundle CHROME_HEADLESS_MODE=false DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL=true DATABASE_TEST_URL=postgresql://hyrax_user:hyrax_password@postgres/hyrax_test?pool=5 @@ -25,9 +26,10 @@ RACK_ENV=development RAILS_ENV=development RAILS_ROOT=.dassie REDIS_HOST=redis +REDIS_PASSWORD=sidekickin REDIS_PROVIDER=SIDEKIQ_REDIS_URL SEED_DASSIE=true -SIDEKIQ_REDIS_URL=redis://redis:6379/0 +SIDEKIQ_REDIS_URL=redis://:sidekickin@redis:6379/0 SOLR_HOST=solr SOLR_PORT=8983 SOLR_TEST_URL=http://solr:8983/solr/hyrax_test diff --git a/.dassie/config/redis.yml b/.dassie/config/redis.yml index d106de12c4..f15c8da92e 100644 --- a/.dassie/config/redis.yml +++ b/.dassie/config/redis.yml @@ -1,9 +1,11 @@ development: host: <%= ENV.fetch('REDIS_HOST', 'localhost') %> port: <%= ENV.fetch('REDIS_PORT', 6379) %> + password: <%= ENV['REDIS_PASSWORD'] %> test: host: <%= ENV.fetch('REDIS_HOST', 'localhost') %> port: <%= ENV.fetch('REDIS_PORT', 6379) %> + password: <%= ENV['REDIS_PASSWORD'] %> production: host: <%= ENV.fetch('REDIS_HOST', 'localhost') %> port: <%= ENV.fetch('REDIS_PORT', 6379) %> diff --git a/.koppie/.env b/.koppie/.env index d7b649332b..a5d09804f1 100644 --- a/.koppie/.env +++ b/.koppie/.env @@ -1,4 +1,5 @@ BUNDLE_GEMFILE=Gemfile.koppie +BUNDLE_PATH=/app/bundle CHROME_HEADLESS_MODE=false DATABASE_CLEANER_ALLOW_REMOTE_DATABASE_URL=true DATABASE_TEST_URL=postgresql://hyrax_user:hyrax_password@postgres/hyrax_test?pool=5 @@ -29,7 +30,7 @@ RAILS_LOG_TO_STDOUT=true RAILS_QUEUE=sidekiq REDIS_HOST=redis REDIS_PASSWORD=sidekickin -REDIS_URL=redis://noop:sidekickin@redis:6379 +REDIS_URL=redis://:sidekickin@redis:6379 SEED_KOPPIE=true SKIP_SELENIUM= SOLR_COLLECTION_NAME=koppie diff --git a/.koppie/config/application.rb b/.koppie/config/application.rb index d209b2b071..f4b43dfc02 100644 --- a/.koppie/config/application.rb +++ b/.koppie/config/application.rb @@ -6,7 +6,7 @@ # you've limited to :test, :development, or :production. Bundler.require(*Rails.groups) -module NuraxPg +module Koppie class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. config.load_defaults 6.0 diff --git a/bin/dev-entrypoint.sh b/bin/dev-entrypoint.sh new file mode 100755 index 0000000000..38659431db --- /dev/null +++ b/bin/dev-entrypoint.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -e + +mkdir -p /app/samvera/hyrax-webapp/tmp/pids +rm -f /app/samvera/hyrax-webapp/tmp/pids/* + +# Copy gems installed in the image to the dev bundle +mkdir -p /app/bundle/ruby/$RUBY_MAJOR.0 +cp -Rn /usr/local/bundle/* /app/bundle/ruby/$RUBY_MAJOR.0 +bundle install + +# Run the command +exec "$@" diff --git a/docker-compose-koppie.yml b/docker-compose-koppie.yml index 316a167883..e2a95f80d3 100644 --- a/docker-compose-koppie.yml +++ b/docker-compose-koppie.yml @@ -6,10 +6,11 @@ services: context: . target: hyrax-engine-dev args: - - EXTRA_APK_PACKAGES=git less chromium + - EXTRA_APK_PACKAGES=git less - APP_PATH=.koppie - BUNDLE_GEMFILE=Gemfile.koppie - image: ghcr.io/samvera/koppie + image: samvera/koppie + entrypoint: dev-entrypoint.sh command: sh -c 'bundle exec puma -v -b tcp://0.0.0.0:3000' stdin_open: true tty: true @@ -30,8 +31,10 @@ services: - 9877:9876 - 1049:1048 volumes: + - ./bin:/app/samvera - .koppie:/app/samvera/hyrax-webapp - .:/app/samvera/hyrax-engine + - bundle:/app/bundle - hyrax-storage:/app/samvera/hyrax-webapp/storage - hyrax-derivatives:/app/samvera/hyrax-webapp/derivatives - hyrax-uploads:/app/samvera/hyrax-webapp/uploads @@ -45,8 +48,11 @@ services: context: . target: hyrax-engine-dev-worker args: + - EXTRA_APK_PACKAGES=git less - APP_PATH=.koppie - image: ghcr.io/samvera/koppie-worker + - BUNDLE_GEMFILE=Gemfile.koppie + image: samvera/koppie-worker + entrypoint: dev-entrypoint.sh command: sh -c 'bundle exec sidekiq' user: root env_file: @@ -58,8 +64,10 @@ services: - redis - solr volumes: + - ./bin:/app/samvera - .koppie:/app/samvera/hyrax-webapp - .:/app/samvera/hyrax-engine + - bundle:/app/bundle - hyrax-storage:/app/samvera/hyrax-webapp/storage - hyrax-derivatives:/app/samvera/hyrax-webapp/derivatives - hyrax-uploads:/app/samvera/hyrax-webapp/uploads @@ -69,22 +77,20 @@ services: - koppie db_migrate: - build: - context: . - target: hyrax-engine-dev - args: - - EXTRA_APK_PACKAGES=git less - - APP_PATH=.koppie - image: ghcr.io/samvera/koppie + image: samvera/koppie user: root env_file: - .koppie/.env + entrypoint: dev-entrypoint.sh command: db-migrate-seed.sh depends_on: - postgres + - solr volumes: + - ./bin:/app/samvera - .koppie:/app/samvera/hyrax-webapp - .:/app/samvera/hyrax-engine + - bundle:/app/bundle - rails-public:/app/samvera/hyrax-webapp/public - rails-tmp:/app/samvera/hyrax-webapp/tmp networks: @@ -106,7 +112,7 @@ services: - koppie postgres: - image: postgres:latest + image: postgres:15 env_file: - .koppie/.env ports: @@ -158,6 +164,7 @@ services: - koppie volumes: + bundle: db: hyrax-storage: hyrax-derivatives: diff --git a/docker-compose.yml b/docker-compose.yml index 4b99ed584d..e1de4ddced 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,8 @@ services: args: - EXTRA_APK_PACKAGES=git less - BUNDLE_GEMFILE=Gemfile.dassie - image: ghcr.io/samvera/dassie + image: samvera/dassie + entrypoint: dev-entrypoint.sh command: sh -c 'bundle exec puma -v -b tcp://0.0.0.0:3000' stdin_open: true tty: true @@ -29,8 +30,10 @@ services: - 3000:3000 - 9876:9876 volumes: + - ./bin:/app/samvera - .dassie:/app/samvera/hyrax-webapp - .:/app/samvera/hyrax-engine + - bundle:/app/bundle - hyrax-derivatives:/app/samvera/hyrax-webapp/derivatives - hyrax-uploads:/app/samvera/hyrax-webapp/uploads - rails-public:/app/samvera/hyrax-webapp/public @@ -42,7 +45,11 @@ services: build: context: . target: hyrax-engine-dev-worker - image: ghcr.io/samvera/dassie-worker + args: + - EXTRA_APK_PACKAGES=git less + - BUNDLE_GEMFILE=Gemfile.dassie + image: samvera/dassie-worker + entrypoint: dev-entrypoint.sh command: sh -c 'bundle exec sidekiq' user: root env_file: @@ -55,8 +62,10 @@ services: - redis - solr volumes: + - ./bin:/app/samvera - .dassie:/app/samvera/hyrax-webapp - .:/app/samvera/hyrax-engine + - bundle:/app/bundle - hyrax-derivatives:/app/samvera/hyrax-webapp/derivatives - hyrax-uploads:/app/samvera/hyrax-webapp/uploads - sidekiq-public:/app/samvera/hyrax-webapp/public @@ -65,21 +74,19 @@ services: - hyrax db_migrate: - build: - context: . - target: hyrax-engine-dev - args: - - EXTRA_APK_PACKAGES=git less - image: ghcr.io/samvera/dassie + image: samvera/dassie user: root env_file: - .dassie/.env + entrypoint: dev-entrypoint.sh command: db-migrate-seed.sh depends_on: - postgres volumes: + - ./bin:/app/samvera - .dassie:/app/samvera/hyrax-webapp - .:/app/samvera/hyrax-engine + - bundle:/app/bundle - rails-public:/app/samvera/hyrax-webapp/public - rails-tmp:/app/samvera/hyrax-webapp/tmp networks: @@ -101,7 +108,7 @@ services: - hyrax postgres: - image: postgres:latest + image: postgres:15 environment: - POSTGRES_USER=hyrax_user - POSTGRES_PASSWORD=hyrax_password @@ -138,14 +145,16 @@ services: - hyrax redis: - image: redis:5-alpine + image: bitnami/redis:6.2 + env_file: + - .dassie/.env volumes: - - redis:/data + - redis:/bitnami/redis/data networks: - hyrax solr: - image: solr:8.11.1 + image: solr:8.11 ports: - 8983:8983 command: @@ -163,6 +172,7 @@ services: - hyrax volumes: + bundle: db: fcrepo: hyrax-derivatives: