Skip to content
CI Pipe

【开源之夏】add dpo trainer and support dpo training for mindnlp #3501

Sign in to view logs

【开源之夏】add dpo trainer and support dpo training for mindnlp

【开源之夏】add dpo trainer and support dpo training for mindnlp #3501

Workflow file for this run

.github/workflows/ci_pipeline.yaml at 8ea1944
# This workflow will install Python dependencies, run tests and lint with a single version of Python
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: CI Pipe
on:
pull_request:
branches: [ "master" ]
paths:
- 'mindnlp/**'
- 'tests/ut/**'
- '!mindnlp/dataset/**'
- '!tests/ut/dataset/**'
- '!docs/**'
- '.github/workflows/**'
push:
branches: [ "master" ]
paths:
- 'mindnlp/**'
- '!mindnlp/dataset/**'
- '!docs/**'
permissions:
contents: read
jobs:
pylint-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install dependencies
run: |
python -m pip install --upgrade pip==24.0
pip install -r requirements/requirements.txt
- name: Install MindSpore
shell: bash
env:
OS: ubuntu-latest
PYTHON: 3.9
run: |
python .github/install_mindspore.py
pip install -r download.txt
- name: Analysing the mindnlp code with pylint
run: |
NUM_CORES=$(nproc)
pylint --jobs=$NUM_CORES mindnlp --rcfile=.github/pylint.conf
ut-test:
needs: pylint-check
strategy:
matrix:
os: [ubuntu-latest, macos-latest, windows-latest]
python: [3.9]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: avoid macos error
if: matrix.os == 'macos-latest'
run: rm -rf $RUNNER_TOOL_CACHE/Python/3.8.17
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip==24.0
pip install -r requirements/requirements.txt
- name: Install MindSpore
shell: bash
env:
OS: ${{ matrix.os }}
PYTHON: ${{ matrix.python }}
run: |
python .github/install_mindspore.py
pip install -r download.txt
- name: Test with pytest
run: |
pytest -c pytest.ini -m 'not download and not gpu_only' --ignore=tests/ut/transformers tests/ut
release-test:
needs: pylint-check
strategy:
matrix:
ms_version: ['2.2.14', '2.3.1']
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install dependencies
run: |
python -m pip install --upgrade pip==24.0
pip install -r requirements/requirements.txt
- name: Install MindSpore
shell: bash
run: |
pip install https://ms-release.obs.cn-north-4.myhuaweicloud.com/${{matrix.ms_version}}/MindSpore/unified/x86_64/mindspore-${{matrix.ms_version}}-cp39-cp39-linux_x86_64.whl --trusted-host ms-release.obs.cn-north-4.myhuaweicloud.com
- name: Test with pytest
run: |
pytest -c pytest.ini -m 'not download and not gpu_only' --ignore=tests/ut/transformers tests/ut
# pytest -c pytest.ini -m 'not download and not gpu_only' tests/ut
transformers-model-test:
needs: pylint-check
strategy:
matrix:
alpha: ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y']
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: 3.9
- name: Install dependencies
run: |
python -m pip install --upgrade pip==24.0
pip install -r requirements/requirements.txt
- name: Install MindSpore
shell: bash
env:
OS: ubuntu-latest
PYTHON: 3.9
run: |
python .github/install_mindspore.py
pip install -r download.txt
- name: Test with pytest
run: |
pytest -vs tests/ut/transformers/models/${{ matrix.alpha }}*/test_modeling*
st-test:
needs: ut-test
strategy:
matrix:
os: [ubuntu-latest, macos-latest]
python: [3.8]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- name: Set up Python
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python }}
- name: Install dependencies
run: |
python -m pip install --upgrade pip==24.0
pip install -r requirements/requirements.txt
- name: Install MindSpore
shell: bash
env:
OS: ${{ matrix.os }}
PYTHON: ${{ matrix.python }}
run: |
python .github/install_mindspore.py
pip install -r download.txt
- name: Test ST with pytest
run: |
pytest -c pytest.ini tests/st
kaggle-gpu-test:
needs: pylint-check
runs-on: ubuntu-latest
if: github.event_name == 'push' && github.repository_owner == 'mindspore-lab'
steps:
- uses: actions/checkout@v3
- name: Set up Python 3.8
uses: actions/setup-python@v4
with:
python-version: 3.8
- uses: "lvyufeng/action-kaggle-gpu-test@latest"
with:
kaggle_username: "${{ secrets.KAGGLE_USERNAME }}"
kaggle_key: "${{ secrets.KAGGLE_API_KEY }}"
kernel_name: mindnlp
kernel_job_file: .github/kaggle_gpu_test.py