Update CI and build process

.github/workflows/release_pypi.yaml

@@ -4,24 +4,98 @@ on:
  release:
  types:
  - created
+
 jobs:
- release-job:
- name: Build and publish on PyPi
+ build_wheels:
+ name: Build wheels on ${{ matrix.os }}
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-20.04, windows-2019, macos-11]
+
+ steps:
+ - uses: actions/checkout@v3
+
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: '3.10'
+
+ - name: Install cibuildwheel
+ run: python -m pip install cibuildwheel==2.12.3
+
+ - name: Set up Rust
+ uses: actions-rs/toolchain@v1
+ with:
+ profile: minimal
+ toolchain: stable
+ override: true
+
+ - name: Build wheels
+ run: python -m cibuildwheel --output-dir wheelhouse
+ env:
+ CIBW_BUILD: cp37-* cp38-* cp39-* cp310-* cp311-* cp312-*
+ CIBW_ARCHS_LINUX: x86_64 i686 aarch64
+ CIBW_ARCHS_WINDOWS: AMD64 x86
+ CIBW_ARCHS_MACOS: x86_64 arm64
+ CIBW_BEFORE_BUILD: pip install setuptools-rust
+ CIBW_ENVIRONMENT: PATH="$HOME/.cargo/bin:$PATH"
+ CIBW_TEST_COMMAND: python -c "import outlines_core; print(outlines_core.__version__)"
+
+ - uses: actions/upload-artifact@v3
+ with:
+ path: ./wheelhouse/*.whl
+ name: wheels
+
+ build_sdist:
+ name: Build source distribution
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.10"
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install setuptools wheel setuptools-rust
+ - name: Build sdist
+ run: python setup.py sdist
+ - uses: actions/upload-artifact@v3
+ with:
+ path: dist/*.tar.gz
+ name: sdist
+
+ release:
+ name: Release to PyPI
+ needs: [build_wheels, build_sdist]
  runs-on: ubuntu-latest
  steps:
- - name: Checkout
- uses: actions/checkout@v2
- - name: Set up Python
- uses: actions/setup-python@v2
- with:
- python-version: "3.10"
- - name: Build SDist and Wheel
- run: ./.github/scripts/build_sdist_and_wheel.sh
- - name: Check that the package version matches the Release name
- run: |
- grep -Rq "^Version: ${GITHUB_REF:10}$" outlines_core.egg-info/PKG-INFO
- - name: Publish to PyPi
- uses: pypa/[email protected]
- with:
- user: __token__
- password: ${{ secrets.PYPI_TOKEN }}
+ - uses: actions/checkout@v3
+ - name: Set up Python
+ uses: actions/setup-python@v4
+ with:
+ python-version: "3.10"
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install setuptools
+ - name: Generate egg-info
+ run: python setup.py egg_info
+ - name: Check that the package version matches the Release name
+ run: |
+ grep -Rq "^Version: ${GITHUB_REF:10}$" *.egg-info/PKG-INFO
+ - uses: actions/download-artifact@v3
+ with:
+ name: wheels
+ path: dist
+ - uses: actions/download-artifact@v3
+ with:
+ name: sdist
+ path: dist
+ - name: Publish to PyPI
+ uses: pypa/[email protected]
+ with:
+ user: __token__
+ password: ${{ secrets.PYPI_SECRET }}

.github/workflows/release_rust.yaml

@@ -0,0 +1,42 @@
+name: Release Rust Package
+
+on:
+ release:
+ types:
+ - created
+
+jobs:
+ release-rust:
+ name: Build and publish on crates.io
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout
+ uses: actions/checkout@v2
+
+ - name: Install Rust
+ uses: actions-rs/toolchain@v1
+ with:
+ profile: minimal
+ toolchain: stable
+
+ - name: Cache Cargo dependencies
+ uses: actions/cache@v2
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ target
+ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}
+
+ - name: Check Cargo.toml version matches Release tag
+ run: |
+ CARGO_VERSION=$(grep '^version =' Cargo.toml | sed 's/.*"\(.*\)".*/\1/')
+ if [ "${GITHUB_REF#refs/tags/v}" != "$CARGO_VERSION" ]; then
+ echo "Version mismatch: Cargo.toml ($CARGO_VERSION) doesn't match Release tag (${GITHUB_REF#refs/tags/v})"
+ exit 1
+ fi
+
+ - name: Publish to crates.io
+ env:
+ CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }}
+ run: cargo publish

.github/workflows/tests.yml

@@ -102,3 +102,31 @@ jobs:
  - uses: actions/checkout@v3
  - name: Build SDist and Wheel
  run: ./.github/scripts/build_sdist_and_wheel.sh
+
+ cargo-test:
+ name: Run Cargo tests
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Install Rust
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ override: true
+ - name: Run cargo test
+ run: cargo test
+
+ cargo-audit:
+ name: Run Cargo audit
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v3
+ - name: Install Rust
+ uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ override: true
+ - name: Install cargo-audit
+ run: cargo install cargo-audit
+ - name: Run cargo audit
+ run: cargo audit

Cargo.toml

@@ -2,6 +2,9 @@
 name = "outlines-core-rs"
 version = "0.1.0"
 edition = "2021"
+description = "Structured Generation"
+license-file = "LICENSE"
+repository = "https://github.com/outlines-dev/outlines-core"
 
 [lib]
 name = "outlines_core_rs"

RELEASE.md

@@ -0,0 +1,11 @@
+# How to release
+
+## Python
+The python package uses the setuptools-scm plugin to automatically determine the version from git tags. When a release is created, it checks the tag and the CI automatically builds and publishes the package to PyPI.
+
+No internvention should be required.
+
+## Rust
+The rust crate is similarly pushed through a Github Action that triggers on a release. But the version is determined by the Cargo.toml file, which has to be updated manually. Generally, the version should be the same as the python package but this isn't a strict requirement.
+
+Currently we fail the rust release if the version in Cargo.toml doesn't match the tag. If that happens, just manually update the Cargo.toml version to match the tag. Push a new commit and rerun the job.

pyproject.toml

@@ -68,7 +68,7 @@ packages = ["outlines_core"]
 package-dir = {"" = "python"}
 
 [tool.setuptools.package-data]
-"outlines" = ["py.typed"]
+"outlines_core" = ["py.typed", "**/*.pyi"]
 
 [tool.setuptools_scm]
 write_to = "python/outlines_core/_version.py"
@@ -105,8 +105,6 @@ module = [
  "datasets.*",
  "setuptools.*",
  "setuptools_rust.*",
- # TODO: Add type info for the Rust extension
- "outlines_core.fsm.outlines_core_rs.*",
 ]
 ignore_missing_imports = true
 

python/outlines_core/fsm/outlines_core_rs.pyi

@@ -0,0 +1,66 @@
+from typing import Dict, List, Optional, Set, Tuple
+
+class FSMInfo:
+ initial: int
+ finals: Set[int]
+ transitions: Dict[Tuple[int, int], int]
+ alphabet_anything_value: int
+ alphabet_symbol_mapping: Dict[str, int]
+
+ def __init__(
+ self,
+ initial: int,
+ finals: Set[int],
+ transitions: Dict[Tuple[int, int], int],
+ alphabet_anything_value: int,
+ alphabet_symbol_mapping: Dict[str, int],
+ ) -> None: ...
+
+def build_regex_from_schema(
+ json: str, whitespace_pattern: Optional[str] = None
+) -> str: ...
+def to_regex(json: Dict, whitespace_pattern: Optional[str] = None) -> str: ...
+def _walk_fsm(
+ fsm_transitions: Dict[Tuple[int, int], int],
+ fsm_initial: int,
+ fsm_finals: Set[int],
+ token_transition_keys: List[int],
+ start_state: int,
+ full_match: bool,
+) -> List[int]: ...
+def state_scan_tokens(
+ fsm_transitions: Dict[Tuple[int, int], int],
+ fsm_initial: int,
+ fsm_finals: Set[int],
+ vocabulary: List[Tuple[str, List[int]]],
+ vocabulary_transition_keys: List[List[int]],
+ start_state: int,
+) -> Set[Tuple[int, int]]: ...
+def get_token_transition_keys(
+ alphabet_symbol_mapping: Dict[str, int],
+ alphabet_anything_value: int,
+ token_str: str,
+) -> List[int]: ...
+def get_vocabulary_transition_keys(
+ alphabet_symbol_mapping: Dict[str, int],
+ alphabet_anything_value: int,
+ vocabulary: List[Tuple[str, List[int]]],
+ frozen_tokens: Set[str],
+) -> List[List[int]]: ...
+def create_fsm_index_end_to_end(
+ fsm_info: FSMInfo,
+ vocabulary: List[Tuple[str, List[int]]],
+ frozen_tokens: frozenset[str],
+) -> Dict[int, Dict[int, int]]: ...
+
+BOOLEAN: str
+DATE: str
+DATE_TIME: str
+INTEGER: str
+NULL: str
+NUMBER: str
+STRING: str
+STRING_INNER: str
+TIME: str
+UUID: str
+WHITESPACE: str