Merge pull request #6 from emarteca/raft

Merging in improvements for flaky test identification
emarteca · Oct 6, 2023 · c052206 · c052206
2 parents 23d018a + 7dc635f
commit c052206
Show file tree

Hide file tree

Showing 25 changed files with 1,724 additions and 283 deletions.
diff --git a/.github/workflows/barbosa23.yml b/.github/workflows/barbosa23.yml
diff --git a/.github/workflows/barbosa23flaky.yml b/.github/workflows/barbosa23flaky.yml
@@ -0,0 +1,36 @@
+name: Test on Barbosa23 JS projects with >=5 flaky tests
+
+on:
+  push
+
+env:
+  PROJECTS_JSON: |
+    { projects: [
+      {"project_url": "appium/appium", "project_sha": "2d124323c5973ef9d3e190f7401e67106886ffd4"},
+      {"project_url": "badges/shields", "project_sha": "14892e3943a4677332618d8b9f584766f7940ee7"},
+      {"project_url": "facebook/react-native", "project_sha": "af99a6890b84713d002fbbd872f10fe2e6304861"},
+      {"project_url": "FlowCrypt/flowcrypt-browser", "project_sha": "92d0188c66572d2c14ef4ed24602b8a58445630c"},
+      {"project_url": "meteor/meteor", "project_sha": "dc38e4325dcd88fb3c6d1be1639680c6ff6f5e80"},
+      {"project_url": "yui/yui3", "project_sha": "25264e3629b1c07fb779d203c4a25c0879ec862c"}
+    ]}
+
+jobs:
+  build-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix-projects: ${{ steps.set-matrix.outputs.matrix-projects }}
+    steps:
+      - id: set-matrix
+        run: |
+          echo "matrix-projects<<__EOF__" >> $GITHUB_OUTPUT
+          echo $PROJECTS_JSON >> $GITHUB_OUTPUT
+          echo "__EOF__" >> $GITHUB_OUTPUT
+  execute:
+    needs: [build-matrix]
+    strategy:
+      matrix: ${{ fromJSON(needs.build-matrix.outputs.matrix-projects) }}
+      fail-fast: false
+    uses: ./.github/workflows/end2endCustomContainers.yml
+    with:
+      project_url: ${{ matrix.projects.project_url }}
+      project_sha: ${{ matrix.projects.project_sha }}
diff --git a/.github/workflows/end2end.yml b/.github/workflows/end2end.yml
@@ -0,0 +1,107 @@
+name: Test NPMFilter End to End on a Project 
+
+on:
+  workflow_dispatch:
+    inputs:
+        project_url:
+            description: 'GitHub suffix of project to test (username/project)'
+            required: true
+            type: string
+        project_sha:
+            description: 'SHA of project to test'
+            required: true
+            type: string
+  workflow_call:
+    inputs:
+        project_url:
+            description: 'GitHub suffix of project to test (username/project)'
+            required: true
+            type: string
+        project_sha:
+            description: 'SHA of project to test'
+            required: true
+            type: string
+jobs:
+  execute:
+    runs-on: self-hosted
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Build NPMFilter container
+        run: docker build -t npmfilter .
+      - name: Run NPMFilter
+        id: run-npm-filter
+        env:
+            SHA: ${{ inputs.project_sha }}
+            URL: ${{ inputs.project_url }}
+            DOCKER_IMAGE: npmfilter:latest
+        run: |
+            IFS="/" read -r -a projectArray <<< "$URL"
+            OrgName=${projectArray[0]} 
+            ProjectName=${projectArray[1]}
+            LogDir=${URL//\//-}
+            echo "LogDir=$LogDir" >> $GITHUB_OUTPUT
+
+            echo "Running NPMFilter on $OrgName/$ProjectName@$SHA"
+
+            mkdir -p docker_configs/
+            cat >docker_configs/debug_filter_config.json <<EOL
+            {
+              "install": {
+                      "timeout": 1000,
+                      "do_install": true
+              },
+              "dependencies": {
+                      "track_deps": false,
+                      "include_dev_deps": false
+              },
+              "build": {
+                      "track_build": true,
+                      "tracked_build_commands": ["build", "compile", "init"],
+                      "timeout": 1000
+              },
+              "test": {
+                      "track_tests": true,
+                      "test_verbose_all_output": {
+                        "do_verbose_tracking": true,
+                        "verbose_json_output_file": "verbose_test_report.json"
+                      },                  
+                      "tracked_test_commands": ["test", "unit", "cov", "ci", "integration", "lint", "travis", "e2e", "bench",
+                                                                        "mocha", "jest", "ava", "tap", "jasmine"],
+                      "timeout": 1000
+              },
+              "meta_info": {
+                      "VERBOSE_MODE": true,
+                      "ignored_commands": ["watch", "debug"],
+                      "ignored_substrings": ["--watch", "nodemon"],
+                      "rm_after_cloning": false,
+                      "scripts_over_code": [ ],
+                      "QL_queries": [ ]
+              }
+            }
+            EOL
+
+            # Run NPMFilter
+            ./runDocker.sh python3 src/diagnose_github_repo.py --repo_link_and_SHA https://github.com/$URL $SHA --config docker_configs/debug_filter_config.json
+
+            # Get tests overview
+            python3 output_proc_scripts/count_tests_run.py npm_filter_docker_results/ > tests-overview.csv
+
+            # Check if tests were found
+            TestData=$(cat tests-overview.csv)
+            IFS="," read -r -a testCount <<< $(python3 output_proc_scripts/count_tests_run.py npm_filter_docker_results/)
+            TestsRun=${testCount[0]}
+            if [ $TestsRun -le 2 ]; then
+                echo "ERROR: No tests found."
+                exit -1
+            else
+                echo "OK: ${TestsRun} tests found!"
+            fi
+
+            echo "LOGNAME=results-${OrgName}-${ProjectName}-${SHA}" >> "$GITHUB_OUTPUT"
+      - name: Upload output
+        uses: actions/upload-artifact@v2
+        with:
+            name: ${{ steps.run-npm-filter.outputs.LOGNAME }}
+            path: npm_filter_docker_results
diff --git a/.github/workflows/end2endCustomContainers.yml b/.github/workflows/end2endCustomContainers.yml
@@ -0,0 +1,112 @@
+name: Test NPMFilter End to End on a Project with custom-built containers per-project 
+
+on:
+  workflow_dispatch:
+    inputs:
+        project_url:
+            description: 'GitHub suffix of project to test (username/project)'
+            required: true
+            type: string
+        project_sha:
+            description: 'SHA of project to test'
+            required: true
+            type: string
+  workflow_call:
+    inputs:
+        project_url:
+            description: 'GitHub suffix of project to test (username/project)'
+            required: true
+            type: string
+        project_sha:
+            description: 'SHA of project to test'
+            required: true
+            type: string
+jobs:
+  execute:
+    runs-on: self-hosted
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Build NPMFilter container
+        run: |
+           if [ -f "project-overrides/${OrgName}-${ProjectName}.sh" ]; then
+            CUSTOM_INSTALL_SCRIPT="--build-arg CUSTOM_INSTALL_SCRIPT=project-overrides/${OrgName}-${ProjectName}.sh"
+           fi
+           docker build -t npmfilter --build-arg REPO_LINK=https://github.com/${{ inputs.project_url }} --build-arg REPO_COMMIT=${{ inputs.project_sha }} $CUSTOM_INSTALL_SCRIPT .
+      - name: Run NPMFilter
+        id: run-npm-filter
+        env:
+            SHA: ${{ inputs.project_sha }}
+            URL: ${{ inputs.project_url }}
+            DOCKER_IMAGE: npmfilter:latest
+        run: |
+            IFS="/" read -r -a projectArray <<< "$URL"
+            OrgName=${projectArray[0]} 
+            ProjectName=${projectArray[1]}
+            LogDir=${URL//\//-}
+            echo "LogDir=$LogDir" >> $GITHUB_OUTPUT
+
+            echo "Running NPMFilter on $OrgName/$ProjectName@$SHA"
+
+            mkdir -p docker_configs/
+            cat >docker_configs/debug_filter_config.json <<EOL
+            {
+              "install": {
+                      "timeout": 1000,
+                      "do_install": true
+              },
+              "dependencies": {
+                      "track_deps": false,
+                      "include_dev_deps": false
+              },
+              "build": {
+                      "track_build": true,
+                      "tracked_build_commands": ["build", "compile", "init"],
+                      "timeout": 1000
+              },
+              "test": {
+                      "track_tests": true,
+                      "tracked_test_commands": ["test", "unit", "cov", "ci", "integration", "lint", "travis", "e2e", "bench",
+                                                                        "mocha", "jest", "ava", "tap", "jasmine"],
+                      "test_verbose_all_output": {
+                        "do_verbose_tracking": true,
+                        "verbose_json_output_file": "verbose_test_report.json"
+                      },                                                                    
+                      "timeout": 1000
+              },
+              "meta_info": {
+                      "VERBOSE_MODE": true,
+                      "ignored_commands": ["watch", "debug"],
+                      "ignored_substrings": ["--watch", "nodemon"],
+                      "rm_after_cloning": false,
+                      "scripts_over_code": [ ],
+                      "QL_queries": [ ]
+              }
+            }
+            EOL
+
+            CUR_DIR=$(pwd)
+            # Run NPMFilter
+            ./runDocker.sh python3 src/diagnose_github_repo.py --repo_link_and_SHA https://github.com/$URL $SHA --config docker_configs/debug_filter_config.json
+
+            # Get tests overview
+            python3 output_proc_scripts/count_tests_run.py npm_filter_docker_results/ > tests-overview.csv
+
+            # Check if tests were found
+            TestData=$(cat tests-overview.csv)
+            IFS="," read -r -a testCount <<< $(python3 output_proc_scripts/count_tests_run.py npm_filter_docker_results/)
+            TestsRun=${testCount[0]}
+            if [ $TestsRun -le 2 ]; then
+                echo "ERROR: No tests found."
+                exit -1
+            else
+                echo "OK: ${TestsRun} tests found!"
+            fi
+
+            echo "LOGNAME=results-${OrgName}-${ProjectName}-${SHA}" >> "$GITHUB_OUTPUT"
+      - name: Upload output
+        uses: actions/upload-artifact@v2
+        with:
+          name: ${{ steps.run-npm-filter.outputs.LOGNAME }}
+          path: npm_filter_docker_results
diff --git a/.github/workflows/smoketest.yml b/.github/workflows/smoketest.yml
@@ -0,0 +1,40 @@
+name: Test NPMFilter End to End on a toy project
+
+on:
+  push:
+
+env:
+  PROJECTS_JSON: |
+    { projects: [
+    {"project_url": "mtiller/ts-jest-sample", "project_sha": "6739c576d4590c53296f3e4fcdf3074e582ae297"},
+    ]}
+
+jobs:
+  build-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix-projects: ${{ steps.set-matrix.outputs.matrix-projects }}
+    steps:
+      - id: set-matrix
+        run: |
+          echo "matrix-projects<<__EOF__" >> $GITHUB_OUTPUT
+          echo $PROJECTS_JSON >> $GITHUB_OUTPUT
+          echo "__EOF__" >> $GITHUB_OUTPUT
+  execute-specialized-container:
+    needs: [build-matrix]
+    strategy:
+      matrix: ${{ fromJSON(needs.build-matrix.outputs.matrix-projects) }}
+      fail-fast: false
+    uses: ./.github/workflows/end2endCustomContainers.yml
+    with:
+      project_url: ${{ matrix.projects.project_url }}
+      project_sha: ${{ matrix.projects.project_sha }}
+  execute-generic-container:
+    needs: [build-matrix]
+    strategy:
+      matrix: ${{ fromJSON(needs.build-matrix.outputs.matrix-projects) }}
+      fail-fast: false
+    uses: ./.github/workflows/end2end.yml
+    with:
+      project_url: ${{ matrix.projects.project_url }}
+      project_sha: ${{ matrix.projects.project_sha }}
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,7 @@ QLDBs/*
 items.json
 *__page_data.html
 *__results.json
+*_verbose_test_report.json
 local_mount/*
 **/node_modules
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,19 +1,45 @@
 FROM ubuntu:latest
 ARG DEBIAN_FRONTEND=noninteractive
 
-RUN apt-get update \
-	&& apt-get -y install --no-install-recommends python3 git unzip vim curl gnupg nodejs npm xz-utils parallel
-
-RUN apt update
-RUN apt -y install python3-pip
-RUN pip3 install bs4 scrapy
+# build arg: setting up for a specific repo? at a specific commit? custom install script?
+ARG REPO_LINK
+ARG REPO_COMMIT
+# placeholder: if this arg isn't specified, copy over the readme file in configs (can't copy no source, RIP)
+ARG CUSTOM_INSTALL_SCRIPT=configs/README.md
 
 RUN mkdir -p /home/npm-filter/results
+RUN mkdir /home/npm-filter/src
+RUN mkdir /home/npm-filter/configs
+
+COPY src /home/npm-filter/src
+# copy the custom install script if it exists
+RUN echo $CUSTOM_INSTALL_SCRIPT
+COPY ${CUSTOM_INSTALL_SCRIPT} configs/ /home/npm-filter/configs/
+# delete the config readme: we don't need this in the docker. and it's a flag for no-custom-install 
+# since the readme is the default for custom install
+RUN rm /home/npm-filter/configs/README.md
+# and name it the custom_install_script
+RUN if [ -f /home/npm-filter/configs/${CUSTOM_INSTALL_SCRIPT} ] ; then mv /home/npm-filter/configs/${CUSTOM_INSTALL_SCRIPT} /home/npm-filter/configs/custom_install_script ; fi
+COPY *.sh /home/npm-filter/
+COPY get_rel_project_reqs.js /home/npm-filter
 
-COPY . /home/npm-filter
+RUN apt-get update \
+	&& apt-get -y install --no-install-recommends python3 git unzip vim curl gnupg xz-utils parallel
+
+RUN apt -y install python3-pip
+RUN pip3 install bs4 scrapy xmltodict pandas
 
 WORKDIR /home/npm-filter
 
 RUN git config --global http.sslVerify "false"
-RUN npm config set strict-ssl false
-RUN ./build.sh
+RUN ./build.sh $REPO_LINK $REPO_COMMIT
+# source the env variables produced by the build script (node version, etc)
+RUN . /envfile
+
+# add a default command for running the tests for repo_link and commit provided
+# this runs in verbose mode
+# need to use ENV instead of ARG in the CMD b/c docker is 10/10
+ENV ENV_REPO_COMMIT=$REPO_COMMIT
+ENV ENV_REPO_LINK=$REPO_LINK
+# gotta source our env vars so the command can run and use npm/node/etc :-)
+CMD . /envfile; ./run_verbose_for_repo_and_config.sh $ENV_REPO_LINK $ENV_REPO_COMMIT