Skip to content

Commit 9d25efc

Browse files
bigabigbigabig
committed
updated checks to build and start ray in a new network if ray code is changed
1 parent f6e4931 commit 9d25efc

File tree

3 files changed

+73
-5
lines changed

3 files changed

+73
-5
lines changed

‎.github/workflows/backend_checks.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,19 +15,46 @@ jobs:
1515
runs-on: self-hosted
1616
env:
1717
API_WORKERS: 1
18+
RAY_HAS_CHANGED: false
1819
steps:
1920
- name: Checkout
2021
uses: actions/checkout@v3
22+
- name: Check for changes in ray_model_worker (pull request)
23+
id: check_ray_model_worker_changes_pr
24+
if: github.event_name == 'pull_request'
25+
run: |
26+
base_branch=$(git merge-base origin/${{ github.event.pull_request.base.ref }} HEAD)
27+
if git diff --name-only $base_branch HEAD | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/'; then
28+
echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV
29+
fi
30+
- name: Check for changes in ray_model_worker (push)
31+
id: check_ray_model_worker_changes_push
32+
if: github.event_name == 'push'
33+
run: |
34+
if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/'; then
35+
echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV
36+
fi
2137
- name: Prepare environment
2238
run: |
2339
./bin/setup-folders.sh
2440
./bin/setup-envs.sh --project_name action-runner --port_prefix 131
2541
sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
2642
echo "MODELS_CACHE_DIR=$HOME/models_cache" >> docker/.env
2743
mkdir -p $HOME/models_cache
44+
if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
45+
sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
46+
sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.ray.yml
47+
sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.yml
48+
echo "SPACY_CACHE_DIR=$HOME/spacy_models" >> docker/.env
49+
mkdir -p $HOME/spacy_models
50+
fi
2851
- name: Build & Start Docker Containers
2952
working-directory: docker
3053
run: |
54+
if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
55+
docker compose -f compose.ray.yml build
56+
docker compose -f compose.ray.yml up --wait
57+
fi
3158
COMPOSE_PROFILES="weaviate,background,backend" docker compose build
3259
COMPOSE_PROFILES="weaviate,background" docker compose up --wait --quiet-pull
3360
- name: Check 1 - pytest runs without errors
@@ -66,6 +93,11 @@ jobs:
6693
working-directory: docker
6794
if: always()
6895
run: |
96+
if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
97+
docker compose -f compose.ray.yml down -v
98+
RAY_IMAGE=uhhlt/dats_ray:$(grep -oP 'DATS_RAY_DOCKER_VERSION=\K.*' .env)
99+
docker rmi $RAY_IMAGE
100+
fi
69101
docker compose down -v --remove-orphans
70102
micromamba env remove -n dats --yes
71103
BACKEND_IMAGE=uhhlt/dats_backend:$(grep -oP 'DATS_BACKEND_DOCKER_VERSION=\K.*' .env)

‎.github/workflows/frontend_checks.yml‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,28 +17,44 @@ jobs:
1717
VITE_APP_SERVER: http://localhost:13120
1818
COMPOSE_PROFILES: "weaviate,background,backend,frontend"
1919
BACKEND_HAS_NEW_REQUIREMENTS: false
20+
RAY_HAS_CHANGED: false
2021
steps:
2122
- uses: actions/checkout@v3
2223
- name: Checkout PR
2324
if: github.event_name == 'pull_request'
2425
env:
2526
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
2627
run: gh pr checkout ${{ github.event.pull_request.number }}
27-
- name: Check for new backend requirements
28-
id: check_backend_requirements
28+
- name: Check for new backend requirements (pull request)
29+
id: check_backend_requirements_pr
2930
if: github.event_name == 'pull_request'
3031
run: |
3132
base_branch=$(git merge-base origin/${{ github.event.pull_request.base.ref }} HEAD)
3233
if git diff --name-only $base_branch HEAD | grep -q -e '^backend/requirements.txt' -e '^backend/environment.yml'; then
3334
echo "BACKEND_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
3435
fi
35-
- name: Check for new backend requirements on push
36-
id: check_backend_requirements2
36+
- name: Check for new backend requirements (push)
37+
id: check_backend_requirements_push
3738
if: github.event_name == 'push'
3839
run: |
3940
if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/requirements.txt' -e '^backend/environment.yml'; then
4041
echo "BACKEND_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
4142
fi
43+
- name: Check for changes in ray_model_worker (pull request)
44+
id: check_ray_model_worker_changes_pr
45+
if: github.event_name == 'pull_request'
46+
run: |
47+
base_branch=$(git merge-base origin/${{ github.event.pull_request.base.ref }} HEAD)
48+
if git diff --name-only $base_branch HEAD | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/'; then
49+
echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV
50+
fi
51+
- name: Check for changes in ray_model_worker (push)
52+
id: check_ray_model_worker_changes_push
53+
if: github.event_name == 'push'
54+
run: |
55+
if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/'; then
56+
echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV
57+
fi
4258
- name: Prepare environment
4359
run: |
4460
./bin/setup-folders.sh
@@ -49,9 +65,20 @@ jobs:
4965
fi
5066
echo "MODELS_CACHE_DIR=$HOME/models_cache" >> docker/.env
5167
mkdir -p $HOME/models_cache
68+
if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
69+
sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
70+
sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.ray.yml
71+
sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.yml
72+
echo "SPACY_CACHE_DIR=$HOME/spacy_models" >> docker/.env
73+
mkdir -p $HOME/spacy_models
74+
fi
5275
- name: Build & Start Docker Containers
5376
working-directory: docker
5477
run: |
78+
if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
79+
docker compose -f compose.ray.yml build
80+
docker compose -f compose.ray.yml up --wait
81+
fi
5582
if [ "${{ env.BACKEND_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
5683
docker compose build dats-backend-api
5784
fi
@@ -83,6 +110,15 @@ jobs:
83110
working-directory: docker
84111
if: always()
85112
run: |
113+
if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
114+
docker compose -f compose.ray.yml down -v
115+
RAY_IMAGE=uhhlt/dats_ray:$(grep -oP 'DATS_RAY_DOCKER_VERSION=\K.*' .env)
116+
docker rmi $RAY_IMAGE
117+
fi
86118
docker compose down -v
119+
if [ "${{ env.BACKEND_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
120+
BACKEND_IMAGE=uhhlt/dats_backend:$(grep -oP 'DATS_BACKEND_DOCKER_VERSION=\K.*' .env)
121+
docker rmi $BACKEND_IMAGE
122+
fi
87123
FRONTEND_IMAGE=uhhlt/dats_frontend:$(grep -oP 'DATS_FRONTEND_DOCKER_VERSION=\K.*' .env)
88124
docker rmi $FRONTEND_IMAGE

‎.github/workflows/ray_checks.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
working-directory: docker
2929
run: |
3030
docker compose -f compose.ray.yml build
31-
docker compose -f compose.ray.yml up --wait --quiet-pull
31+
docker compose -f compose.ray.yml up --wait
3232
- name: Cleanup
3333
working-directory: docker
3434
if: always()

0 commit comments

Comments
 (0)