Use BOSS indexing in DBGSuccinct + make RowDiff independent #4499
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: MetaGraph CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| tags: | |
| - 'v*' | |
| pull_request: | |
| env: | |
| REGISTRY: ghcr.io | |
| IMAGE_NAME: ${{ github.repository }} | |
| jobs: | |
| Linux: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| alphabet: [DNA, Protein] | |
| build_type: [Debug, Release] | |
| compiler: [g++-8, g++-9, g++-10, g++-11] | |
| build_static: [OFF, ON] | |
| with_avx: [OFF, ON] | |
| include: | |
| - compiler: g++-8 | |
| cxx: g++-8 | |
| cc: gcc-8 | |
| - compiler: g++-9 | |
| cxx: g++-9 | |
| cc: gcc-9 | |
| - compiler: g++-10 | |
| cxx: g++-10 | |
| cc: gcc-10 | |
| - compiler: g++-11 | |
| cxx: g++-11 | |
| cc: gcc-11 | |
| exclude: | |
| - alphabet: Protein | |
| compiler: g++-9 | |
| - compiler: g++-8 | |
| build_static: ON | |
| - compiler: g++-9 | |
| build_static: ON | |
| - compiler: g++-10 | |
| build_static: ON | |
| - build_type: Debug | |
| build_static: ON | |
| - with_avx: OFF | |
| build_static: OFF | |
| name: Linux (${{ matrix.alphabet }}, ${{ matrix.build_type }}, ${{ matrix.compiler }}${{ matrix.build_static == 'ON' && ', static' || '' }}${{ matrix.with_avx == 'OFF' && ', noAVX' || '' }}) | |
| steps: | |
| - uses: actions/checkout@v1 | |
| - name: checkout submodules | |
| run: git submodule update --init --recursive | |
| - name: install dependencies | |
| run: | | |
| sudo apt-get update | |
| # If clang, use the default version. Otherwise the compiler install with apt-get. | |
| if [[ "${{ matrix.compiler }}" != "clang" ]]; then | |
| sudo apt-get install ${{ matrix.compiler }} | |
| fi | |
| sudo apt-get install libbz2-dev libjemalloc-dev libboost-all-dev libdeflate-dev | |
| if [ "${{ matrix.build_static }}" = "ON" ]; then | |
| # for static builds, compile curl ourselves | |
| git clone https://github.com/curl/curl.git | |
| mkdir curl/_build | |
| cd curl/_build | |
| cmake -DBUILD_SHARED_LIBS=off .. | |
| make -j 2 | |
| sudo make install | |
| cd .. | |
| fi | |
| echo "CC=$(which ${{ matrix.cc }})" >> $GITHUB_ENV | |
| echo "CXX=$(which ${{ matrix.cxx }})" >> $GITHUB_ENV | |
| - name: install dependencies for integration tests | |
| run: | | |
| pip3 install parameterized | |
| sudo apt-get install python3-venv | |
| - name: install sdsl-lite | |
| run: | | |
| cd metagraph/external-libraries/sdsl-lite | |
| ./install.sh $(pwd) | |
| - name: configure | |
| run: | | |
| mkdir metagraph/build | |
| cd metagraph/build | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
| -DBUILD_STATIC=${{ matrix.build_static }} \ | |
| -DWITH_AVX=${{ matrix.with_avx }} -DWITH_MSSE42=${{ matrix.with_avx }} \ | |
| -DCMAKE_DBG_ALPHABET=${{ matrix.alphabet }} .. | |
| - name: build metagraph | |
| run: cd metagraph/build && make -j 2 metagraph | |
| - name: build unit tests | |
| if: matrix.build_static == 'OFF' | |
| run: cd metagraph/build && make -j 2 unit_tests | |
| - name: build other | |
| if: matrix.build_static == 'OFF' | |
| run: cd metagraph/build && make -j 2 | |
| # for potential release | |
| - name: rename binary | |
| if: matrix.with_avx == 'OFF' | |
| run: mv metagraph/build/metagraph_${{ matrix.alphabet }} metagraph/build/metagraph_${{ matrix.alphabet }}_noAVX | |
| - name: upload static binary | |
| if: ${{ matrix.build_static == 'ON' && matrix.compiler == 'g++-11' }} | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: metagraph_${{ matrix.alphabet }}${{ matrix.with_avx == 'OFF' && '_noAVX' || '' }}_linux_x86 | |
| path: metagraph/build/metagraph_${{ matrix.alphabet }}${{ matrix.with_avx == 'OFF' && '_noAVX' || '' }} | |
| - name: run unit tests | |
| if: matrix.build_static == 'OFF' | |
| run: | | |
| export LD_LIBRARY_PATH="/usr/local/lib/:$LD_LIBRARY_PATH" | |
| cd metagraph/build && ./unit_tests | |
| - name: run integration tests | |
| uses: nick-fields/retry@v2 | |
| with: | |
| timeout_minutes: 1000 | |
| max_attempts: 5 | |
| retry_on: error | |
| command: | | |
| export LD_LIBRARY_PATH="/usr/local/lib/:$LD_LIBRARY_PATH" | |
| cd metagraph/build && make check | |
| MacOS: | |
| runs-on: macos-latest | |
| strategy: | |
| matrix: | |
| alphabet: [DNA] | |
| build_type: [Debug, Release] | |
| platform: [x64, arm64] | |
| steps: | |
| - uses: actions/checkout@v1 | |
| - name: checkout submodules | |
| run: git submodule update --init --recursive | |
| - name: install dependencies | |
| run: brew install bzip2 libomp boost jemalloc autoconf automake libtool libdeflate | |
| - name: install dependencies for integration tests | |
| run: pip3 install parameterized --break-system-packages --user | |
| - name: install sdsl-lite | |
| run: | | |
| cd metagraph/external-libraries/sdsl-lite | |
| ./install.sh $(pwd) | |
| - name: configure | |
| run: | | |
| mkdir metagraph/build | |
| cd metagraph/build | |
| cmake -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \ | |
| -DCMAKE_DBG_ALPHABET=${{ matrix.alphabet }} \ | |
| -DWITH_AVX=OFF -DWITH_MSSE42=OFF .. | |
| - name: build metagraph | |
| run: cd metagraph/build && make -j 2 metagraph | |
| - name: build unit tests | |
| run: cd metagraph/build && make -j 2 unit_tests | |
| - name: build other | |
| run: cd metagraph/build && make -j 2 | |
| - name: run unit tests | |
| run: cd metagraph/build && ./unit_tests | |
| - name: run integration tests | |
| run: cd metagraph/build && make check | |
| Metagraph-Workflows: | |
| name: Test metagraph workflows | |
| runs-on: ubuntu-20.04 | |
| needs: [Linux] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set up Python 3.8 | |
| uses: actions/setup-python@v1 | |
| with: | |
| python-version: 3.8 | |
| - name: fetch static binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: setup metagraph binary | |
| run: | | |
| sudo ln -s $(pwd)/artifacts/metagraph_DNA_linux_x86/metagraph_DNA /usr/local/bin/metagraph | |
| sudo chmod +rx /usr/local/bin/metagraph | |
| /usr/local/bin/metagraph --help | |
| metagraph --help | |
| - name: Install python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install pytest | |
| pip install -r metagraph/workflows/requirements.txt | |
| - name: Test metagraph-workflows pytest | |
| run: | | |
| cd metagraph/workflows | |
| pytest | |
| Build-and-Push-Docker: | |
| # adapted from https://docs.github.com/en/actions/guides/publishing-docker-images#publishing-images-to-github-packages | |
| if: github.event_name != 'pull_request' | |
| needs: [Linux, Metagraph-Workflows] | |
| runs-on: ubuntu-20.04 | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v2 | |
| - name: checkout submodules | |
| run: git submodule update --init --recursive | |
| - name: Log in to the Container registry | |
| uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 | |
| with: | |
| registry: ${{ env.REGISTRY }} | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Extract metadata (tags, labels) for Docker | |
| id: meta | |
| uses: docker/metadata-action@v3 | |
| with: | |
| images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} | |
| - name: Build and push Docker image | |
| uses: docker/build-push-action@v2 | |
| with: | |
| context: . | |
| push: true | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| Release: | |
| name: Create Release | |
| if: startsWith(github.ref, 'refs/tags/v') | |
| runs-on: ubuntu-latest | |
| needs: [Linux, Metagraph-Workflows] | |
| steps: | |
| - uses: actions/checkout@v1 | |
| - name: checkout submodules | |
| run: git submodule update --init --recursive | |
| - name: fetch static binary | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: artifacts | |
| - name: Tag name | |
| id: tag_name | |
| run: | | |
| echo ::set-output name=TAG::${GITHUB_REF#refs/tags/} | |
| echo ::set-output name=VER::${GITHUB_REF#refs/tags/v} | |
| # from https://stackoverflow.com/a/61919791/6869319 | |
| - name: read current version | |
| id: read_version | |
| run: | | |
| content=`cat package.json` | |
| # the following lines are only required for multi line json | |
| content="${content//'%'/'%25'}" | |
| content="${content//$'\n'/'%0A'}" | |
| content="${content//$'\r'/'%0D'}" | |
| # end of optional handling for multi line json | |
| echo "::set-output name=packageJson::$content" | |
| - name: build source archive including submodules | |
| run: | | |
| [[ v${{fromJson(steps.read_version.outputs.packageJson).version}} == ${{ steps.tag_name.outputs.TAG }} ]] \ | |
| || (echo "::error file=package.json::Version in package.json (v${{fromJson(steps.read_version.outputs.packageJson).version}}) does not match tag ${{ steps.tag_name.outputs.TAG }}. Update package.json and set the tag again."; \ | |
| exit 1) | |
| mkdir -p artifacts/sources | |
| tag=${{ steps.tag_name.outputs.TAG }} | |
| bash .github/workflows/git-archive-all.sh --format tar.gz artifacts/sources/${tag}-sources-with-submodules.tar.gz | |
| bash .github/workflows/git-archive-all.sh --format zip artifacts/sources/${tag}-sources-with-submodules.zip | |
| - name: create release with static binaries | |
| uses: svenstaro/upload-release-action@v2 | |
| with: | |
| release_name: Version ${{ steps.tag_name.outputs.VER }} | |
| body: | | |
| The pre-compiled binaries (only for Linux) are statically linked and hence don't require additional libraries to be installed. | |
| The *_noAVX binaries are compiled without the modern AVX2 and MSSE4.2 instructions and should be used when the processor does not support these instructions. | |
| file: 'artifacts/*/*' | |
| file_glob: true | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| tag: ${{ github.ref }} | |
| prerelease: true |