packer #14
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Submitty DB Check | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| push: | |
| paths: | |
| - migration/** | |
| workflow_dispatch: | |
| jobs: | |
| db-check: | |
| runs-on: ubuntu-20.04 | |
| env: | |
| PGPASSWORD: submitty_dbuser | |
| MAIN_INSTALL_PATH: ${{ github.WORKSPACE }}/main | |
| MAIN_DATA_PATH: ${{ github.WORKSPACE }}/main/var | |
| MAIN_REPO_PATH: ${{ github.WORKSPACE }}/main/GIT_CHECKOUT | |
| PR_INSTALL_PATH: ${{ github.WORKSPACE }}/branch | |
| PR_DATA_PATH: ${{ github.WORKSPACE }}/branch/var | |
| PR_REPO_PATH: ${{ github.WORKSPACE }}/branch/GIT_CHECKOUT | |
| services: | |
| postgres-main: | |
| image: postgres:12 | |
| env: | |
| POSTGRES_PASSWORD: submitty_dbuser | |
| POSTGRES_USER: postgres | |
| TZ: America/New_York | |
| ports: | |
| - 5432 | |
| postgres-branch: | |
| image: postgres:12 | |
| env: | |
| POSTGRES_PASSWORD: submitty_dbuser | |
| POSTGRES_USER: postgres | |
| TZ: America/New_York | |
| ports: | |
| - 5432 | |
| steps: | |
| - name: Setup Dependencies | |
| run: | | |
| sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' | |
| wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - | |
| sudo apt-get update | |
| sudo apt-get autoremove -y postgresql-14 postgresql-client-common | |
| sudo apt-get install -y postgresql-12 | |
| sudo rm /etc/postgresql/14 -rf | |
| - name: Checkout Current Branch | |
| uses: actions/checkout@v3 | |
| with: | |
| path: branch/GIT_CHECKOUT/Submitty | |
| fetch-depth: 0 | |
| - name: Find Merge Base | |
| run: | | |
| echo MBASE=$(git --git-dir ${PR_REPO_PATH}/Submitty/.git merge-base \ | |
| -a origin/main ${{ github.sha }}) >> $GITHUB_ENV | |
| - name: Checkout main Branch | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ env.MBASE || 'main' }} | |
| path: main/GIT_CHECKOUT/Submitty | |
| - name: Download Required Packages | |
| run: | | |
| pip install -r <( | |
| grep -e "tzlocal" -e "sqlalchemy" -e "ruamel" \ | |
| -e "psycopg2" -e "docker" \ | |
| ${MAIN_REPO_PATH}/Submitty/.setup/pip/system_requirements.txt | |
| ) | |
| pip install ${MAIN_REPO_PATH}/Submitty/python_submitty_utils | |
| - name: Create Users and Groups | |
| run: | # Create users and groups | |
| sudo -E python3 ${MAIN_REPO_PATH}/Submitty/.setup/bin/create_untrusted_users.py | |
| sudo addgroup submitty_daemonphp | |
| sudo addgroup submitty_daemoncgi | |
| sudo addgroup submitty_course_builders | |
| sudo addgroup submitty_phpgrp | |
| sudo adduser submitty_php --disabled-password --no-create-home | |
| sudo adduser submitty_cgi --disabled-password --no-create-home | |
| sudo adduser submitty_cgi submitty_phpgrp --no-create-home | |
| sudo adduser submitty_php shadow --no-create-home | |
| sudo adduser submitty_cgi shadow --no-create-home | |
| sudo adduser submitty_daemon --disabled-password --no-create-home | |
| sudo adduser submitty_php submitty_daemonphp --no-create-home | |
| sudo adduser submitty_daemon submitty_daemonphp --no-create-home | |
| sudo adduser submitty_cgi submitty_daemoncgi --no-create-home | |
| sudo adduser submitty_daemon submitty_daemoncgi --no-create-home | |
| sudo adduser submitty_daemon docker --no-create-home | |
| sudo useradd -p $(openssl passwd -1 submitty_dbuser) submitty_dbuser | |
| - name: Configure Submitty | |
| run: | | |
| configure_submitty() { | |
| # $1->MAIN_INSTALL_PATH, $2->MAIN_DATA_PATH, $3->MAIN_REPO_PATH, $4->DB_PORT | |
| mkdir -vp "$2" && chmod -vR 777 "$2" | |
| mkdir -vp "$1/config" && chmod -vR 777 "$1/config" | |
| mkdir -vp "$1/site" && chmod -vR 777 "$1/site" | |
| mkdir -vp "$2/courses" && chmod -vR 777 "$2/courses" | |
| ( | |
| echo "localhost" # database host | |
| echo "$4" # database port | |
| echo "" # global db user | |
| echo "submitty_dbuser" # global db pass | |
| echo "" # course db user | |
| echo "submitty_dbuser" # course db pass | |
| echo "" # timezone | |
| echo "" # default locale | |
| echo "http://localhost" # submitty url | |
| echo "" # vcs url | |
| echo "" # institution name | |
| echo "" # sysadmin email | |
| echo "" # where to report | |
| echo "1" # PamAuth | |
| echo "" # sysadmin username | |
| echo "n" # email notification | |
| ) | tee /dev/stderr \ | |
| | sudo -E python3 "$3/Submitty/.setup/CONFIGURE_SUBMITTY.py" \ | |
| --install-dir "$1" --data-dir "$2" --debug | |
| # Copy necessary files to the installation path | |
| cp -vr "$3/Submitty/sbin" "$1" | |
| sudo chmod -vR 777 "$1/config" | |
| sudo chmod -vR 777 "$1/sbin" | |
| } | |
| configure_submitty "${MAIN_INSTALL_PATH}" "${MAIN_DATA_PATH}" \ | |
| "${MAIN_REPO_PATH}" "${{ job.services.postgres-main.ports['5432'] }}" | |
| configure_submitty "${PR_INSTALL_PATH}" "${PR_DATA_PATH}" \ | |
| "${PR_REPO_PATH}" "${{ job.services.postgres-branch.ports['5432'] }}" | |
| - name: Create Databases | |
| run: | | |
| psql -d postgres -h localhost -U postgres -p ${{ job.services.postgres-main.ports['5432'] }} \ | |
| -c "CREATE ROLE submitty_dbuser WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'submitty_dbuser'"\ | |
| -c "CREATE ROLE submitty_course_dbuser WITH LOGIN PASSWORD 'submitty_dbuser'" | |
| psql -d postgres -h localhost -U submitty_dbuser -p ${{ job.services.postgres-main.ports['5432'] }} \ | |
| -c "CREATE DATABASE submitty" | |
| psql -d postgres -h localhost -U postgres -p ${{ job.services.postgres-branch.ports['5432'] }} \ | |
| -c "CREATE ROLE submitty_dbuser WITH SUPERUSER CREATEDB CREATEROLE LOGIN PASSWORD 'submitty_dbuser'"\ | |
| -c "CREATE ROLE submitty_course_dbuser WITH LOGIN PASSWORD 'submitty_dbuser'" | |
| psql -d postgres -h localhost -U submitty_dbuser -p ${{ job.services.postgres-branch.ports['5432'] }} \ | |
| -c "CREATE DATABASE submitty" | |
| - name: Create Database Dumper | |
| run: | # the original dumper requires a `su`, which is not needed when psql as a service | |
| ( | |
| echo 'pg_dump -Osx -d "$1" -f "$2" -h localhost -p "$3" -U postgres ' | |
| echo 'sed -i -e "'"/SELECT pg_catalog.set_config('search_path'/d"'"\' | |
| echo ' -e "/-- Dumped \(from\|by\)/d" \' | |
| echo ' -e "/SET default_table_access_method = /d" \' | |
| echo ' -e "s/FOR EACH ROW EXECUTE FUNCTION/FOR EACH ROW EXECUTE PROCEDURE/g" "$2"' | |
| ) > dbdump.sh | |
| chmod +x dbdump.sh | |
| echo M=$([ "$(date +%-m)" -lt "7" ] && echo 's' || echo 'f') >> $GITHUB_ENV | |
| echo Y=$(date +%y) >> $GITHUB_ENV | |
| mkdir -vp ${GITHUB_WORKSPACE}/dumped/main | |
| mkdir -vp ${GITHUB_WORKSPACE}/dumped/branch | |
| - name: Setup Databases and Courses for main | |
| run: | # migrate main according to dumped master database (assume it is correct) | |
| python3 ${MAIN_REPO_PATH}/Submitty/migration/run_migrator.py -e master migrate --initial | |
| sudo ln -vs /usr/bin/true /usr/bin/ldapadd | |
| sudo -E python3 ${MAIN_REPO_PATH}/Submitty/.setup/bin/setup_sample_courses.py \ | |
| blank --no_grading --data_dir ${MAIN_DATA_PATH} --install_dir ${MAIN_INSTALL_PATH} | |
| sudo chmod -vR 777 ${MAIN_DATA_PATH}/courses | |
| - name: Apply New Migrations from Current Branch | |
| run: | | |
| python3 ${PR_REPO_PATH}/Submitty/migration/run_migrator.py \ | |
| -e master -e course -c ${MAIN_INSTALL_PATH}/config migrate | |
| - name: Setup Databases and Courses for Current Branch | |
| run: | | |
| python3 ${PR_REPO_PATH}/Submitty/migration/run_migrator.py -e master migrate --initial | |
| sudo -E python3 ${PR_REPO_PATH}/Submitty/.setup/bin/setup_sample_courses.py \ | |
| blank --no_grading --data_dir ${PR_DATA_PATH} --install_dir ${PR_INSTALL_PATH} | |
| sudo chmod -vR 777 ${PR_DATA_PATH}/courses | |
| - name: Dump Migrated Databases | |
| run: | | |
| ./dbdump.sh "submitty" "${GITHUB_WORKSPACE}/dumped/main/master.sql"\ | |
| "${{ job.services.postgres-main.ports['5432'] }}" | |
| ./dbdump.sh "submitty_${M}${Y}_blank" "${GITHUB_WORKSPACE}/dumped/main/course.sql"\ | |
| "${{ job.services.postgres-main.ports['5432'] }}" | |
| ./dbdump.sh "submitty" "${GITHUB_WORKSPACE}/dumped/branch/master.sql"\ | |
| "${{ job.services.postgres-branch.ports['5432'] }}" | |
| ./dbdump.sh "submitty_${M}${Y}_blank" "${GITHUB_WORKSPACE}/dumped/branch/course.sql"\ | |
| "${{ job.services.postgres-branch.ports['5432'] }}" | |
| - name: Compare Migrated master Databases | |
| if: always() | |
| run: | | |
| diff --color ${GITHUB_WORKSPACE}/dumped/main/master.sql \ | |
| ${GITHUB_WORKSPACE}/dumped/branch/master.sql || { | |
| echo "::error:: Master databases are not identical, please check your migration" | |
| exit 1 | |
| } | |
| - name: Compare Migrated course Databases | |
| if: always() | |
| run: | | |
| diff --color ${GITHUB_WORKSPACE}/dumped/main/course.sql \ | |
| ${GITHUB_WORKSPACE}/dumped/branch/course.sql || { | |
| echo "::error:: Course databases are not identical, please check your migration" | |
| exit 1 | |
| } | |
| - name: Compare Migrated master Database with Provided Dump | |
| if: always() | |
| run: | | |
| diff --color ${GITHUB_WORKSPACE}/dumped/branch/master.sql \ | |
| ${PR_REPO_PATH}/Submitty/migration/migrator/data/submitty_db.sql || { | |
| echo "::error:: Wrong provided dump, please check your database dumps" | |
| exit 1 | |
| } | |
| - name: Compare Migrated course Database with Provided Dump | |
| if: always() | |
| run: | | |
| diff --color ${GITHUB_WORKSPACE}/dumped/branch/course.sql \ | |
| ${PR_REPO_PATH}/Submitty/migration/migrator/data/course_tables.sql || { | |
| echo "::error:: Wrong provided dump, please check your database dumps" | |
| exit 1 | |
| } | |
| - name: Pack Artifacts | |
| if: failure() | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: dumped_database | |
| path: ${{ github.WORKSPACE }}/dumped/**/*.sql | |
| retention-days: 3 |