From 2e8498d668d7e5d3d05d3ff7884de6de224e00c3 Mon Sep 17 00:00:00 2001 From: erikkaum Date: Fri, 30 Aug 2024 10:23:25 +0200 Subject: [PATCH] Add package publishing workflows for Rust and Python --- .github/workflows/release_pypi.yaml | 110 +++++++++++++++--- .github/workflows/release_rust.yaml | 42 +++++++ .github/workflows/tests.yml | 28 +++++ Cargo.toml | 3 + RELEASE.md | 11 ++ pyproject.toml | 4 +- python/outlines_core/fsm/outlines_core_rs.pyi | 66 +++++++++++ 7 files changed, 243 insertions(+), 21 deletions(-) create mode 100644 .github/workflows/release_rust.yaml create mode 100644 RELEASE.md create mode 100644 python/outlines_core/fsm/outlines_core_rs.pyi diff --git a/.github/workflows/release_pypi.yaml b/.github/workflows/release_pypi.yaml index eb4a9a7c..dd975b6c 100644 --- a/.github/workflows/release_pypi.yaml +++ b/.github/workflows/release_pypi.yaml @@ -4,24 +4,98 @@ on: release: types: - created + jobs: - release-job: - name: Build and publish on PyPi + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-20.04, windows-2019, macos-11] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.10' + + - name: Install cibuildwheel + run: python -m pip install cibuildwheel==2.12.3 + + - name: Set up Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + + - name: Build wheels + run: python -m cibuildwheel --output-dir wheelhouse + env: + CIBW_BUILD: cp37-* cp38-* cp39-* cp310-* cp311-* cp312-* + CIBW_ARCHS_LINUX: x86_64 i686 aarch64 + CIBW_ARCHS_WINDOWS: AMD64 x86 + CIBW_ARCHS_MACOS: x86_64 arm64 + CIBW_BEFORE_BUILD: pip install setuptools-rust + CIBW_ENVIRONMENT: PATH="$HOME/.cargo/bin:$PATH" + CIBW_TEST_COMMAND: python -c "import outlines_core; print(outlines_core.__version__)" + + - uses: actions/upload-artifact@v3 + with: + path: ./wheelhouse/*.whl + name: wheels + + build_sdist: + name: Build source distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools wheel setuptools-rust + - name: Build sdist + run: python setup.py sdist + - uses: actions/upload-artifact@v3 + with: + path: dist/*.tar.gz + name: sdist + + release: + name: Release to PyPI + needs: [build_wheels, build_sdist] runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.10" - - name: Build SDist and Wheel - run: ./.github/scripts/build_sdist_and_wheel.sh - - name: Check that the package version matches the Release name - run: | - grep -Rq "^Version: ${GITHUB_REF:10}$" outlines_core.egg-info/PKG-INFO - - name: Publish to PyPi - uses: pypa/gh-action-pypi-publish@v1.4.2 - with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install setuptools + - name: Generate egg-info + run: python setup.py egg_info + - name: Check that the package version matches the Release name + run: | + grep -Rq "^Version: ${GITHUB_REF:10}$" *.egg-info/PKG-INFO + - uses: actions/download-artifact@v3 + with: + name: wheels + path: dist + - uses: actions/download-artifact@v3 + with: + name: sdist + path: dist + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@v1.5.0 + with: + user: __token__ + password: ${{ secrets.PYPI_SECRET }} diff --git a/.github/workflows/release_rust.yaml b/.github/workflows/release_rust.yaml new file mode 100644 index 00000000..1cc3b162 --- /dev/null +++ b/.github/workflows/release_rust.yaml @@ -0,0 +1,42 @@ +name: Release Rust Package + +on: + release: + types: + - created + +jobs: + release-rust: + name: Build and publish on crates.io + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + + - name: Cache Cargo dependencies + uses: actions/cache@v2 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + + - name: Check Cargo.toml version matches Release tag + run: | + CARGO_VERSION=$(grep '^version =' Cargo.toml | sed 's/.*"\(.*\)".*/\1/') + if [ "${GITHUB_REF#refs/tags/v}" != "$CARGO_VERSION" ]; then + echo "Version mismatch: Cargo.toml ($CARGO_VERSION) doesn't match Release tag (${GITHUB_REF#refs/tags/v})" + exit 1 + fi + + - name: Publish to crates.io + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} + run: cargo publish diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e5f0010..77593611 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -102,3 +102,31 @@ jobs: - uses: actions/checkout@v3 - name: Build SDist and Wheel run: ./.github/scripts/build_sdist_and_wheel.sh + + cargo-test: + name: Run Cargo tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Run cargo test + run: cargo test + + cargo-audit: + name: Run Cargo audit + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Install cargo-audit + run: cargo install cargo-audit + - name: Run cargo audit + run: cargo audit diff --git a/Cargo.toml b/Cargo.toml index 05fec5a8..6862cfe3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,9 @@ name = "outlines-core-rs" version = "0.1.0" edition = "2021" +description = "Structured Generation" +license-file = "LICENSE" +repository = "https://github.com/outlines-dev/outlines-core" [lib] name = "outlines_core_rs" diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 00000000..e22d1247 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,11 @@ +# How to release + +## Python +The python package uses the setuptools-scm plugin to automatically determine the version from git tags. When a release is created, it checks the tag and the CI automatically builds and publishes the package to PyPI. + +No internvention should be required. + +## Rust +The rust crate is similarly pushed through a Github Action that triggers on a release. But the version is determined by the Cargo.toml file, which has to be updated manually. Generally, the version should be the same as the python package but this isn't a strict requirement. + +Currently we fail the rust release if the version in Cargo.toml doesn't match the tag. If that happens, just manually update the Cargo.toml version to match the tag. Push a new commit and rerun the job. diff --git a/pyproject.toml b/pyproject.toml index bfc4cab5..e8255045 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ packages = ["outlines_core"] package-dir = {"" = "python"} [tool.setuptools.package-data] -"outlines" = ["py.typed"] +"outlines_core" = ["py.typed", "**/*.pyi"] [tool.setuptools_scm] write_to = "python/outlines_core/_version.py" @@ -105,8 +105,6 @@ module = [ "datasets.*", "setuptools.*", "setuptools_rust.*", - # TODO: Add type info for the Rust extension - "outlines_core.fsm.outlines_core_rs.*", ] ignore_missing_imports = true diff --git a/python/outlines_core/fsm/outlines_core_rs.pyi b/python/outlines_core/fsm/outlines_core_rs.pyi new file mode 100644 index 00000000..f970980b --- /dev/null +++ b/python/outlines_core/fsm/outlines_core_rs.pyi @@ -0,0 +1,66 @@ +from typing import Dict, List, Optional, Set, Tuple + +class FSMInfo: + initial: int + finals: Set[int] + transitions: Dict[Tuple[int, int], int] + alphabet_anything_value: int + alphabet_symbol_mapping: Dict[str, int] + + def __init__( + self, + initial: int, + finals: Set[int], + transitions: Dict[Tuple[int, int], int], + alphabet_anything_value: int, + alphabet_symbol_mapping: Dict[str, int], + ) -> None: ... + +def build_regex_from_schema( + json: str, whitespace_pattern: Optional[str] = None +) -> str: ... +def to_regex(json: Dict, whitespace_pattern: Optional[str] = None) -> str: ... +def _walk_fsm( + fsm_transitions: Dict[Tuple[int, int], int], + fsm_initial: int, + fsm_finals: Set[int], + token_transition_keys: List[int], + start_state: int, + full_match: bool, +) -> List[int]: ... +def state_scan_tokens( + fsm_transitions: Dict[Tuple[int, int], int], + fsm_initial: int, + fsm_finals: Set[int], + vocabulary: List[Tuple[str, List[int]]], + vocabulary_transition_keys: List[List[int]], + start_state: int, +) -> Set[Tuple[int, int]]: ... +def get_token_transition_keys( + alphabet_symbol_mapping: Dict[str, int], + alphabet_anything_value: int, + token_str: str, +) -> List[int]: ... +def get_vocabulary_transition_keys( + alphabet_symbol_mapping: Dict[str, int], + alphabet_anything_value: int, + vocabulary: List[Tuple[str, List[int]]], + frozen_tokens: Set[str], +) -> List[List[int]]: ... +def create_fsm_index_end_to_end( + fsm_info: FSMInfo, + vocabulary: List[Tuple[str, List[int]]], + frozen_tokens: frozenset[str], +) -> Dict[int, Dict[int, int]]: ... + +BOOLEAN: str +DATE: str +DATE_TIME: str +INTEGER: str +NULL: str +NUMBER: str +STRING: str +STRING_INNER: str +TIME: str +UUID: str +WHITESPACE: str