Skip to content

Commit f6cd9e5

Browse files
bigabigbigabig
authored
Merge pull request #484 from uhh-lt/mwp_v1
Switch to main branch
2 parents b3ba3dc + 0d05888 commit f6cd9e5

File tree

1,312 files changed

+187638
-5
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,312 files changed

+187638
-5
lines changed

‎.github/workflows/backend_checks.yml‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
name: backend checks
2+
on:
3+
push:
4+
branches:
5+
- main
6+
- mwp_v1
7+
pull_request:
8+
paths:
9+
- "backend/**"
10+
- ".github/**"
11+
- "docker/**"
12+
- "tools/**"
13+
14+
jobs:
15+
backend-checks:
16+
runs-on: self-hosted
17+
env:
18+
API_WORKERS: 1
19+
RAY_CONFIG: "config_gpu.yaml"
20+
JWT_SECRET: ${{ secrets.JWT_SECRET }}
21+
steps:
22+
- name: Checkout
23+
uses: actions/checkout@v3
24+
- name: Prepare environment
25+
run: |
26+
./bin/setup-folders.sh
27+
./bin/setup-envs.sh --project_name action-runner --port_prefix 131
28+
sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
29+
sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
30+
echo "SPACY_CACHE_DIR=$HOME/spacy_models" >> docker/.env
31+
echo "MODELS_CACHE_DIR=$HOME/models_cache" >> docker/.env
32+
mkdir -p $HOME/spacy_models
33+
mkdir -p $HOME/models_cache
34+
- name: Build & Start Docker Containers
35+
working-directory: docker
36+
run: |
37+
COMPOSE_PROFILES="weaviate,ray,background,backend" docker compose build
38+
COMPOSE_PROFILES="weaviate,ray,background" docker compose up --wait --quiet-pull
39+
- name: Check 1 - pytest runs without errors
40+
working-directory: docker
41+
run: |
42+
docker compose run dats-backend-api /opt/envs/dats/bin/python -m pytest
43+
- name: Check 2 - Database migrates without errors database
44+
working-directory: docker
45+
run: |
46+
docker compose run -e PYTHONPATH='/dats_code/src' dats-backend-api /opt/envs/dats/bin/python migration/run_migrations.py
47+
- name: Check 3 - Database schema is up-to-date after migration
48+
working-directory: docker
49+
run: |
50+
docker compose run dats-backend-api /opt/envs/dats/bin/alembic check
51+
- name: Start Remaining Docker Containers
52+
working-directory: docker
53+
run: |
54+
COMPOSE_PROFILES="weaviate,ray,background,backend" docker compose up --wait --quiet-pull
55+
- name: Check 4 - Test End-2-End importer script
56+
working-directory: tools/importer
57+
env:
58+
TESTDATA_PASSWORD: ${{ secrets.TESTDATA_PASSWORD }}
59+
run: |
60+
pip install -r requirements.txt
61+
wget -q http://ltdata1.informatik.uni-hamburg.de/dwts/totalitarismo.zip
62+
unzip -q -P "$TESTDATA_PASSWORD" totalitarismo.zip
63+
python dats_importer.py --input_dir json --backend_url http://localhost:13120/ --is_json --doctype text
64+
python dats_importer.py --input_dir images --backend_url http://localhost:13120/ --doctype image
65+
- name: Check 5 - pyright runs without errors
66+
run: |
67+
micromamba env create -f backend/environment.yml --yes
68+
micromamba run -n dats pip install -r backend/src/app/preprocessing/ray_model_worker/requirements.txt
69+
micromamba run -n dats pip install ray==2.32.0
70+
micromamba run -n dats pyright
71+
- name: Cleanup
72+
working-directory: docker
73+
if: always()
74+
run: |
75+
docker compose down -v --remove-orphans
76+
micromamba env remove -n dats --yes
77+
BACKEND_IMAGE=uhhlt/dats_backend:$(grep -oP 'DATS_BACKEND_DOCKER_VERSION=\K.*' .env)
78+
RAY_IMAGE=uhhlt/dats_ray:$(grep -oP 'DATS_RAY_DOCKER_VERSION=\K.*' .env)
79+
docker rmi $BACKEND_IMAGE $RAY_IMAGE

‎.github/workflows/frontend_checks.yml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
name: frontend checks
2+
on:
3+
push:
4+
branches:
5+
- main
6+
- mwp_v1
7+
pull_request:
8+
paths:
9+
- ".github/**"
10+
- "docker/**"
11+
- "frontend/**"
12+
13+
jobs:
14+
frontend-checks:
15+
runs-on: self-hosted
16+
env:
17+
API_WORKERS: 1
18+
VITE_APP_SERVER: http://localhost:13120
19+
COMPOSE_PROFILES: "weaviate,ray,background,backend,frontend"
20+
RAY_CONFIG: "config_gpu.yaml"
21+
JWT_SECRET: ${{ secrets.JWT_SECRET }}
22+
BACKEND_HAS_NEW_REQUIREMENTS: false
23+
RAY_HAS_NEW_REQUIREMENTS: false
24+
steps:
25+
- uses: actions/checkout@v3
26+
- name: Checkout PR
27+
if: github.event_name == 'pull_request'
28+
env:
29+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
30+
run: gh pr checkout ${{ github.event.pull_request.number }}
31+
- name: Check for new backend requirements
32+
id: check_backend_requirements
33+
if: github.event_name == 'pull_request'
34+
run: |
35+
base_branch=$(git merge-base origin/${{ github.event.pull_request.base.ref }} HEAD)
36+
if git diff --name-only $base_branch HEAD | grep -q -e '^backend/requirements.txt' -e '^backend/environment.yml'; then
37+
echo "BACKEND_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
38+
fi
39+
- name: Check for new backend requirements on push
40+
id: check_backend_requirements2
41+
if: github.event_name == 'push'
42+
run: |
43+
if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/requirements.txt' -e '^backend/environment.yml'; then
44+
echo "BACKEND_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
45+
fi
46+
- name: Check for new ray requirements
47+
id: check_ray_requirements
48+
if: github.event_name == 'pull_request'
49+
run: |
50+
base_branch=$(git merge-base origin/${{ github.event.pull_request.base.ref }} HEAD)
51+
if git diff --name-only $base_branch HEAD | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/requirements.txt'; then
52+
echo "RAY_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
53+
fi
54+
- name: Check for new ray requirements on push
55+
id: check_ray_requirements2
56+
if: github.event_name == 'push'
57+
run: |
58+
if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/requirements.txt'; then
59+
echo "RAY_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
60+
fi
61+
- name: Prepare environment
62+
run: |
63+
./bin/setup-folders.sh
64+
./bin/setup-envs.sh --project_name action-runner --port_prefix 131
65+
sed -i 's/\(DATS_FRONTEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
66+
if [ "${{ env.RAY_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
67+
sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
68+
fi
69+
if [ "${{ env.BACKEND_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
70+
sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
71+
fi
72+
echo "SPACY_CACHE_DIR=$HOME/spacy_models" >> docker/.env
73+
echo "MODELS_CACHE_DIR=$HOME/models_cache" >> docker/.env
74+
mkdir -p $HOME/spacy_models
75+
mkdir -p $HOME/models_cache
76+
- name: Build & Start Docker Containers
77+
working-directory: docker
78+
run: |
79+
if [ "${{ env.RAY_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
80+
docker compose build ray
81+
fi
82+
if [ "${{ env.BACKEND_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
83+
docker compose build dats-backend-api
84+
fi
85+
docker compose build dats-frontend
86+
docker compose up --wait --quiet-pull
87+
- name: Setup node
88+
uses: actions/setup-node@v4
89+
with:
90+
node-version: 20
91+
cache: npm
92+
cache-dependency-path: frontend/package-lock.json
93+
- name: Install dependencies
94+
working-directory: frontend
95+
run: npm ci
96+
- name: Check 1 - api client is up-to-date
97+
working-directory: frontend
98+
run: |
99+
npm run update-api
100+
npm run generate-api
101+
- name: Add and commit changes
102+
uses: EndBug/add-and-commit@v9
103+
if: github.event_name == 'pull_request'
104+
with:
105+
add: "frontend/src/api"
106+
message: "Update OpenAPI spec and client"
107+
pathspec_error_handling: exitImmediately
108+
default_author: github_actions
109+
- name: Cleanup
110+
working-directory: docker
111+
if: always()
112+
run: |
113+
docker compose down -v
114+
FRONTEND_IMAGE=uhhlt/dats_frontend:$(grep -oP 'DATS_FRONTEND_DOCKER_VERSION=\K.*' .env)
115+
docker rmi $FRONTEND_IMAGE

‎.github/workflows/release.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
name: Release
2+
3+
on:
4+
push:
5+
tags:
6+
- "v*.*.*"
7+
8+
jobs:
9+
build_release:
10+
name: build_release
11+
runs-on: ubuntu-latest
12+
steps:
13+
- name: checkout
14+
uses: actions/checkout@v3
15+
16+
- name: Set up Docker Buildx
17+
uses: docker/setup-buildx-action@v2
18+
19+
- name: Login to Docker Hub
20+
uses: docker/login-action@v2
21+
with:
22+
username: ${{ secrets.DOCKERHUB_USERNAME }}
23+
password: ${{ secrets.DOCKERHUB_TOKEN }}
24+
25+
- name: Get version from tag
26+
id: extract_version
27+
run: |
28+
echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"
29+
shell: bash
30+
31+
- name: Free Disk Space (Ubuntu)
32+
uses: jlumbroso/free-disk-space@main
33+
with:
34+
tool-cache: false
35+
docker-images: false
36+
android: true
37+
dotnet: true
38+
haskell: true
39+
large-packages: true
40+
swap-storage: true
41+
42+
- name: Build and push backend container
43+
uses: docker/build-push-action@v4
44+
with:
45+
context: ./backend
46+
push: true
47+
tags: |
48+
"uhhlt/dats_backend:${{ steps.extract_version.outputs.version }}"
49+
- name: Prepare environment
50+
working-directory: frontend
51+
run: cp .env.production.example .env.production
52+
- name: Build and push frontend container
53+
uses: docker/build-push-action@v4
54+
with:
55+
context: ./frontend
56+
push: true
57+
tags: |
58+
"uhhlt/dats_frontend:${{ steps.extract_version.outputs.version }}"
59+
- name: Build and push ray container
60+
uses: docker/build-push-action@v4
61+
with:
62+
context: ./backend/src/app/preprocessing/ray_model_worker
63+
push: true
64+
tags: |
65+
"uhhlt/dats_ray:${{ steps.extract_version.outputs.version }}"
66+
67+
- name: Create Release
68+
uses: ncipollo/release-action@v1
69+
with:
70+
generateReleaseNotes: true
71+
deploy_demo:
72+
name: deploy_demo
73+
needs: build_release
74+
runs-on: ubuntu-latest
75+
steps:
76+
- name: Checkout
77+
id: checkout
78+
uses: actions/checkout@v3
79+
80+
- name: Write SSH config & key
81+
id: write_ssh_keys
82+
env:
83+
SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
84+
SSH_CONFIG: ${{ secrets.SSH_CONFIG }}
85+
run: |
86+
install -m 600 -D /dev/null ~/.ssh/id_ed25519
87+
echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519
88+
echo "$SSH_CONFIG" > ~/.ssh/config
89+
shell: bash
90+
91+
- name: Add server to known hosts
92+
env:
93+
SSH_JUMP_HOST: ${{ secrets.SSH_JUMP_HOST }}
94+
run: |
95+
ssh-keyscan -H $SSH_JUMP_HOST >> ~/.ssh/known_hosts
96+
shell: bash
97+
98+
- name: Deploy
99+
id: deploy
100+
env:
101+
SSH_HOST: ${{ secrets.SSH_HOST }}
102+
run: |
103+
scp -o StrictHostKeyChecking=no bin/deploy.sh $SSH_HOST:~/deploy.sh
104+
ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=5 -o StrictHostKeyChecking=no $SSH_HOST './deploy.sh'
105+
ssh -o StrictHostKeyChecking=no $SSH_HOST 'rm ./deploy.sh'
106+
shell: bash

‎.pre-commit-config.yaml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
ci:
2+
skip: [eslint, pyright]
3+
repos:
4+
- repo: https://github.com/pre-commit/pre-commit-hooks
5+
rev: v4.6.0
6+
hooks:
7+
- id: check-yaml
8+
- id: end-of-file-fixer
9+
- id: requirements-txt-fixer
10+
- id: trailing-whitespace
11+
- repo: https://github.com/astral-sh/ruff-pre-commit
12+
# Ruff version.
13+
rev: v0.6.7
14+
hooks:
15+
# Run the linter.
16+
- id: ruff
17+
args: [--fix]
18+
# Run the formatter.
19+
- id: ruff-format
20+
- repo: local
21+
hooks:
22+
- id: pyright
23+
name: "Pyright"
24+
types: [python]
25+
entry: bash -c 'ENV_NAME=dats source backend/_activate_current_env.sh && pyright $@'
26+
language: system
27+
- repo: https://github.com/pre-commit/mirrors-eslint
28+
rev: v9.11.0
29+
hooks:
30+
- id: eslint
31+
files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx
32+
types: [file]
33+
args: [--max-warnings=0]
34+
additional_dependencies:
35+
- "@tanstack/[email protected]"
36+
- [email protected]
37+
- [email protected]
38+
- [email protected]
39+
- [email protected]
40+
- repo: https://github.com/pre-commit/mirrors-prettier
41+
rev: "v4.0.0-alpha.8"
42+
hooks:
43+
- id: prettier
44+
files: \.([jt]sx?|json)$ # *.js, *.jsx, *.ts, *.tsx, *.json
45+
args: [
46+
--print-width,
47+
"120",
48+
# Without this, prettier will create a `node_modules` folder in the repository root
49+
"--cache-location",
50+
"frontend/node_modules/.cache/prettier",
51+
]

‎.vscode/extensions.json‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
{
2+
"recommendations": [
3+
"ms-azuretools.vscode-docker",
4+
// git
5+
"eamodio.gitlens",
6+
"github.vscode-github-actions",
7+
"github.copilot",
8+
// backend
9+
"ms-python.python",
10+
"ms-python.debugpy",
11+
"ms-python.vscode-pylance",
12+
"charliermarsh.ruff",
13+
// frontend
14+
"dbaeumer.vscode-eslint",
15+
"esbenp.prettier-vscode"
16+
]
17+
}

0 commit comments

Comments
 (0)