Improved performance of the github workflow (#236)

* Event trigger * Workflow dispatch * Changed dockerfile to work locally * Chose 1 random test for every category * Test compile * Testing parallelization of the tests * Fix space * No anchor * Fixed back andy branch * Added checkstyle step and fixed typo in workflow step * Runs on os * Mvn cache * Added back install for weblab tests * Commented clean install * Added back mvn install step * Mvn install -pl * Fixed pipeline * Pl andy * Skiptests for install * Fixed install script and fail-fast * Rename weblab runner tests * Update step title --------- Co-authored-by: Martin Mladenov <[email protected]>
SERG-Delft · Jul 20, 2023 · c856ece · c856ece
1 parent 2448a0f
commit c856ece
Show file tree

Hide file tree

Showing 3 changed files with 135 additions and 58 deletions.
diff --git a/.github/scripts/assignments_test_docker.py b/.github/scripts/assignments_test_docker.py
@@ -1,4 +1,4 @@
-import os, re, sys
+import os, re, sys, random
 
 def get_directories(basedir):
  return [os.path.join(basedir, dir) for dir in os.listdir(basedir) \
@@ -21,50 +21,51 @@ def get_directories(basedir):
 
 pipeline_failed = False
 for category_dir in get_directories(home_dir):
- for assignment_dir in get_directories(category_dir):
- # Remove the contents of the output directory.
- os.system(f'rm -r {output_dir}')
-
- # Remove the contents of the test directory.
- os.system(f'rm -r {test_dir}/*')
-
- # Write environment file
- with open(f'{test_dir}/.env', 'w') as envfile:
- envfile.write('TASK_MODE=FULL_WITH_HINTS')
-
- # Copy the assignment to the test folder.
- os.chdir(assignment_dir)
- os.system(f'cp ./config/Configuration.java {test_dir}/test.txt')
- os.system(f'cp ./solution/*.java {test_dir}/solution.txt')
- os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt')
- # Copy resources
- os.system('find . -type f | ' +
- 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' +
- 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' +
- 'xargs -i cp --parents {} ' + f'{test_dir}/')
-
- # Switch to Docker directory
- os.chdir(docker_dir)
-
- # Run `andy` on the current assignment.
- output = os.popen('make github-ci.test').read()
-
- re_score = re.search('Final grade: [0-9]+', output)
- score = int(re_score.group().split()[2]) if re_score else -1
- re_andy_version = re.search('Andy v.+', output)
- andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version"
-
- # Print the score for the assignment.
- print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100')
-
- # Update the `pipeline_failed` variable.
- if score != 100:
- print(output)
- pipeline_failed = True
-
- if expected_andy_version not in andy_version:
- print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}')
- pipeline_failed = True
+ #for assignment_dir in get_directories(category_dir):
+ assignment_dir = random.choice(get_directories(category_dir));
+ # Remove the contents of the output directory.
+ os.system(f'rm -r {output_dir}')
+
+ # Remove the contents of the test directory.
+ os.system(f'rm -r {test_dir}/*')
+
+ # Write environment file
+ with open(f'{test_dir}/.env', 'w') as envfile:
+ envfile.write('TASK_MODE=FULL_WITH_HINTS')
+
+ # Copy the assignment to the test folder.
+ os.chdir(assignment_dir)
+ os.system(f'cp ./config/Configuration.java {test_dir}/test.txt')
+ os.system(f'cp ./solution/*.java {test_dir}/solution.txt')
+ os.system(f'cp ./src/main/java/delft/*.java {test_dir}/library.txt')
+ # Copy resources
+ os.system('find . -type f | ' +
+ 'grep -i -v "^\./src/" | grep -i -v "\./config/Configuration.java" | ' +
+ 'grep -i -v "^\./pom.xml$" | grep -i -v "^\./solution/" | grep -i -v "^\./README.md$" | ' +
+ 'xargs -i cp --parents {} ' + f'{test_dir}/')
+
+ # Switch to Docker directory
+ os.chdir(docker_dir)
+
+ # Run `andy` on the current assignment.
+ output = os.popen('make github-ci.test').read()
+
+ re_score = re.search('Final grade: [0-9]+', output)
+ score = int(re_score.group().split()[2]) if re_score else -1
+ re_andy_version = re.search('Andy v.+', output)
+ andy_version = re_andy_version.group() if re_andy_version else "Unknown Andy version"
+
+ # Print the score for the assignment.
+ print(f'{andy_version} | {assignment_dir.split("/")[-2]}/{assignment_dir.split("/")[-1]}: {score}/100')
+
+ # Update the `pipeline_failed` variable.
+ if score != 100:
+ print(output)
+ pipeline_failed = True
+
+ if expected_andy_version not in andy_version:
+ print(f'Error: Unexpected Andy version {andy_version}, expected {expected_andy_version}')
+ pipeline_failed = True
 
 if pipeline_failed:
  sys.exit('Some assignments do not have 100/100.')
diff --git a/.github/workflows/assignments_test_docker.yml b/.github/workflows/assignments_test_docker.yml
@@ -1,5 +1,5 @@
 name: assignments_test_docker
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
 env:
  # PR: github.sha is the SHA of the PR, not commit, we can't check this out
  GHA_SHA: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
@@ -30,7 +30,7 @@ jobs:
  run: |
  make
 
- - name: Run the reference solutions of all assignments and verify the scores are 100/100
+ - name: Run the reference solutions of one assignment per category and verify the scores are 100/100
  env:
  # github.repository contains owner, we don't want that
  REPO_NAME: ${{ github.event.repository.name }}

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -1,7 +1,33 @@
 name: tests
-on: [push, pull_request]
+on: [push, pull_request, workflow_dispatch]
+
 jobs:
- run_tests:
+ checkstyle:
+ runs-on: ubuntu-22.04
+ strategy:
+ fail-fast: true
+ steps:
+ - name: Checkout the repository
+ uses: actions/checkout@v3
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'zulu'
+ java-version: 17
+
+ - name: Cache Maven packages
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2
+ key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ matrix.job.os }}-m2
+
+ - name: Run checkstyle
+ run: mvn checkstyle:check
+
+ run_unit_tests:
+ needs: checkstyle
  runs-on: ${{ matrix.job.os }}
  strategy:
  fail-fast: false
@@ -28,20 +54,70 @@ jobs:
  key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }}
  restore-keys: ${{ matrix.job.os }}-m2
 
- - name: Compile and install everything
- run: mvn clean install -DskipTests
-
  - name: Run Andy unit tests
  run: mvn test -pl andy -DexcludedGroups=assignments
 
+ run_assignment_tests:
+ needs: checkstyle
+ runs-on: ${{ matrix.job.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ job:
+ - os: ubuntu-22.04
+ - os: macos-12
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
+
+ steps:
+ - name: Checkout the repository
+ uses: actions/checkout@v3
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'zulu'
+ java-version: 17
+
+ - name: Cache Maven packages
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2
+ key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ matrix.job.os }}-m2
+
  - name: Run Andy in all assignments
  run: mvn test -pl andy -Dgroups=assignments
 
- - name: Run Weblab runner tests
- run: mvn test -pl weblab-runner -DexcludedGroups=selenium
+ run_weblab_runner_tests:
+ needs: checkstyle
+ runs-on: ${{ matrix.job.os }}
+ strategy:
+ fail-fast: false
+ matrix:
+ job:
+ - os: ubuntu-22.04
+ - os: macos-12
+ if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name
 
-# - name: Run Weblab selenium tests
-# run: mvn test -pl weblab-runner -Dgroups=selenium
+ steps:
+ - name: Checkout the repository # Anchors not available in github actions :(
+ uses: actions/checkout@v3
 
- - name: Run Checkstyle
- run: mvn checkstyle:check
+ - name: Set up JDK 17
+ uses: actions/setup-java@v3
+ with:
+ distribution: 'zulu'
+ java-version: 17
+
+ - name: Cache Maven packages
+ uses: actions/cache@v3
+ with:
+ path: ~/.m2
+ key: ${{ matrix.job.os }}-m2-${{ hashFiles('**/pom.xml') }}
+ restore-keys: ${{ matrix.job.os }}-m2
+
+ - name: Compile and install everything
+ run: mvn install -pl andy -DskipTests
+
+ - name: Run Weblab runner tests
+ run: mvn test -pl weblab-runner -DexcludedGroups=selenium