ci-test.yml

# Github action definitions for ci-test with PRs.

name: tfx-ci-test
on:
  pull_request:
    branches: [ master ]
    paths-ignore:
      - '**.md'
      - 'docs/**'

jobs:
  build:
    if: github.actor != 'copybara-service[bot]'
    runs-on: ubuntu-latest
    timeout-minutes: 60

    steps:
    - uses: actions/checkout@v2
    - name: Get Changed Files
      id: changed_files
      uses: trilom/file-changes-action@v1.2.4
      with:
        fileOutput: ' '
    - name: Select files to check
      run: |
        # Filter out non-python files.
        (cat $HOME/files_added.txt; echo; cat $HOME/files_modified.txt) | tr ' ' '\n' | grep '\.py$' > py_files.txt || true
        # Filter out non-test python files and e2e or integration tests.
        cat py_files.txt | grep '_test\.py$' | grep -v _e2e_ | grep -v integration | grep -v 'examples/' > py_test_files.txt || true
        # Select proto files.
        (cat $HOME/files_added.txt; echo; cat $HOME/files_modified.txt) | tr ' ' '\n' | grep '\.proto$' > proto_files.txt || true

    - name: Set up Python 3.9
      uses: actions/setup-python@v1
      with:
        python-version: 3.9

    - name: Set up Bazel 5.3.0
      run: |
        # Instruction from https://docs.bazel.build/versions/master/install-ubuntu.html
        curl -sSL https://github.com/bazelbuild/bazel/releases/download/5.3.0/bazel-5.3.0-installer-linux-x86_64.sh -o bazel_installer.sh
        chmod +x bazel_installer.sh
        sudo ./bazel_installer.sh

    - name: Cache pip
      uses: actions/cache@v2
      with:
        # This path is specific to Ubuntu
        path: ~/.cache/pip
        # Look to see if there is a cache hit for the corresponding setup.py + TFX version 
        key: ${{ runner.os }}-pip-${{ hashFiles('tfx/dependencies.py') }}-
        restore-keys: |
          ${{ runner.os }}-pip-

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip wheel
        # TODO(b/232490018): Cython need to be installed separately to build pycocotools.
        python -m pip install Cython -c ./test_constraints.txt
        TFX_DEPENDENCY_SELECTOR=NIGHTLY pip install -c ./test_constraints.txt --extra-index-url https://pypi-nightly.tensorflow.org/simple --pre --editable .[all]

    - name: Run unit tests
      shell: bash
      run: |
        [ ! -s "py_test_files.txt" ] || cat py_test_files.txt | xargs -I {} python {}

    - name: Lint with protolint
      continue-on-error: true
      env:
        PROTOLINT_VERSION: 0.25.1
      shell: bash
      run: |
        curl -sSOL https://github.com/yoheimuta/protolint/releases/download/v${PROTOLINT_VERSION}/protolint_${PROTOLINT_VERSION}_Linux_x86_64.tar.gz
        tar zxf protolint_${PROTOLINT_VERSION}_Linux_x86_64.tar.gz
        echo "[NOTE] This linter is currently EXPERIMENTAL.======================================="
        echo "Please contact reviewers for existing lint errors or false negative errors."
        echo "===================================================================================="
        [ ! -s "proto_files.txt" ] || cat proto_files.txt | xargs -I {} ./protolint {}

    - name: Lint with pylint
      continue-on-error: true
      shell: bash
      run: |
        pip install pylint
        echo "[NOTE] This linter is currently EXPERIMENTAL.======================================="
        echo "Please contact reviewers for existing lint errors or false negative errors."
        echo "Feel free to send PRs for pylintrc in the root directory of the repository if needed."
        echo "===================================================================================="
        [ ! -s "py_files.txt" ] || pylint $(cat py_files.txt | tr '\n' ' ')