Deploy a1a5ab8 to gh-pages

.cargo/config.toml

@@ -0,0 +1,8 @@
+[build]
+rustflags = ["-C", "target-cpu=native"]
+
+[target.wasm32-unknown-unknown]
+rustflags = ["-C", "target-feature=+simd128"]
+
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "target-feature=-avx,-avx2"]

.github/workflows/book-cd.yml

@@ -0,0 +1,40 @@
+name: Deploy Rust book
+on:
+ push:
+ branches:
+ - main
+
+jobs:
+ deploy:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write # To push a branch 
+ pull-requests: write # To create a PR from that branch
+ steps:
+ - uses: actions/checkout@v3
+ with:
+ fetch-depth: 0
+ - name: Install latest mdbook
+ run: |
+ tag=$(curl 'https://api.github.com/repos/rust-lang/mdbook/releases/latest' | jq -r '.tag_name')
+ url="https://github.com/rust-lang/mdbook/releases/download/${tag}/mdbook-${tag}-x86_64-unknown-linux-gnu.tar.gz"
+ mkdir mdbook
+ curl -sSL $url | tar -xz --directory=./mdbook
+ echo `pwd`/mdbook >> $GITHUB_PATH
+ - name: Deploy GitHub Pages
+ run: |
+ # This assumes your book is in the root of your repository.
+ # Just add a `cd` here if you need to change to another directory.
+ cd candle-book
+ mdbook build
+ git worktree add gh-pages
+ git config user.name "Deploy from CI"
+ git config user.email ""
+ cd gh-pages
+ # Delete the ref to avoid keeping history.
+ git update-ref -d refs/heads/gh-pages
+ rm -rf *
+ mv ../book/* .
+ git add .
+ git commit -m "Deploy $GITHUB_SHA to gh-pages"
+ git push --force --set-upstream origin gh-pages

.github/workflows/book.yml

@@ -0,0 +1,29 @@
+name: CI
+on: 
+ pull_request:
+
+jobs:
+ test:
+ name: Test candle-book
+ runs-on: ubuntu-latest
+ permissions:
+ contents: write # To push a branch 
+ pull-requests: write # To create a PR from that branch
+ steps:
+ - uses: actions/checkout@master
+ - name: Install Rust
+ run: |
+ rustup set profile minimal
+ rustup toolchain install stable
+ rustup default stable
+ - name: Install latest mdbook
+ run: |
+ tag=$(curl 'https://api.github.com/repos/rust-lang/mdbook/releases/latest' | jq -r '.tag_name')
+ url="https://github.com/rust-lang/mdbook/releases/download/${tag}/mdbook-${tag}-x86_64-unknown-linux-gnu.tar.gz"
+ mkdir bin
+ curl -sSL $url | tar -xz --directory=bin
+ echo "$(pwd)/bin" >> $GITHUB_PATH
+ - name: Run tests
+ run: cd candle-book && cargo build && mdbook test -L ../target/debug/deps/
+
+

.github/workflows/ci_cuda.yaml

@@ -0,0 +1,87 @@
+name: CI / cuda
+
+on:
+ workflow_dispatch:
+ pull_request:
+
+jobs:
+ start-runner:
+ name: Start self-hosted EC2 runner
+ runs-on: ubuntu-latest
+ env:
+ AWS_REGION: us-east-1
+ EC2_AMI_ID: ami-03cfed9ea28f4b002
+ EC2_INSTANCE_TYPE: g5.xlarge
+ EC2_SUBNET_ID: subnet-931b34f5,subnet-ecb993cd,subnet-943dc2d8,subnet-45371f1a,subnet-ee93e0df,subnet-fddc3dfc
+ EC2_SECURITY_GROUP: sg-030175c435ac141d6
+ outputs:
+ label: ${{ steps.start-ec2-runner.outputs.label }}
+ ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+ steps:
+ - name: Configure AWS credentials
+ uses: aws-actions/configure-aws-credentials@v1
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ env.AWS_REGION }}
+ - name: Start EC2 runner
+ id: start-ec2-runner
+ uses: philschmid/philschmid-ec2-github-runner@main
+ with:
+ mode: start
+ github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+ ec2-image-id: ${{ env.EC2_AMI_ID }}
+ ec2-instance-type: ${{ env.EC2_INSTANCE_TYPE }}
+ subnet-id: ${{ env.EC2_SUBNET_ID }}
+ security-group-id: ${{ env.EC2_SECURITY_GROUP }}
+ aws-resource-tags: > # optional, requires additional permissions
+ [
+ {"Key": "Name", "Value": "ec2-tgi-github-runner"},
+ {"Key": "GitHubRepository", "Value": "${{ github.repository }}"}
+ ]
+
+ test-cuda:
+ concurrency:
+ group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+ needs: start-runner # required to start the main job when the runner is ready
+ runs-on: ${{ needs.start-runner.outputs.label }} # run the job on the newly created runner
+ permissions:
+ contents: write
+ packages: write
+ # This is used to complete the identity challenge
+ # with sigstore/fulcio when running outside of PRs.
+ id-token: write
+ security-events: write
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v3
+ - name: Install Rust Stable
+ run: curl https://sh.rustup.rs -sSf | sh -s -- -y
+ - uses: Swatinem/rust-cache@v2
+ - run: apt-get update -y && apt-get install libssl-dev -y
+ - name: Test (cuda)
+ run: PATH=$PATH:/usr/local/cuda-11.8/bin/ /root/.cargo/bin/cargo test --features cuda
+ stop-runner:
+ name: Stop self-hosted EC2 runner
+ needs:
+ - start-runner
+ - test-cuda
+ runs-on: ubuntu-latest
+ env:
+ AWS_REGION: us-east-1
+ if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+ steps:
+ - name: Configure AWS credentials
+ uses: aws-actions/configure-aws-credentials@v1
+ with:
+ aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+ aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ aws-region: ${{ env.AWS_REGION }}
+ - name: Stop EC2 runner
+ uses: philschmid/philschmid-ec2-github-runner@main
+ with:
+ mode: stop
+ github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }}
+ label: ${{ needs.start-runner.outputs.label }}
+ ec2-instance-id: ${{ needs.start-runner.outputs.ec2-instance-id }}

.github/workflows/rust-ci.yml

@@ -0,0 +1,78 @@
+on: 
+ push:
+ branches: 
+ - main
+ pull_request:
+
+name: Continuous integration
+
+jobs:
+ check:
+ name: Check
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-latest, windows-latest, macOS-latest]
+ rust: [stable]
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions-rs/toolchain@v1
+ with:
+ profile: minimal
+ toolchain: ${{ matrix.rust }}
+ override: true
+ - uses: actions-rs/cargo@v1
+ with:
+ command: check
+ args: --workspace
+
+ test:
+ name: Test Suite
+ runs-on: ${{ matrix.os }}
+ strategy:
+ matrix:
+ os: [ubuntu-latest, windows-latest, macOS-latest]
+ rust: [stable]
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions-rs/toolchain@v1
+ with:
+ profile: minimal
+ toolchain: ${{ matrix.rust }}
+ override: true
+ - uses: actions-rs/cargo@v1
+ with:
+ command: test
+ args: --workspace
+
+ fmt:
+ name: Rustfmt
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions-rs/toolchain@v1
+ with:
+ profile: minimal
+ toolchain: stable
+ override: true
+ - run: rustup component add rustfmt
+ - uses: actions-rs/cargo@v1
+ with:
+ command: fmt
+ args: --all -- --check
+
+ clippy:
+ name: Clippy
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ - uses: actions-rs/toolchain@v1
+ with:
+ profile: minimal
+ toolchain: stable
+ override: true
+ - run: rustup component add clippy
+ - uses: actions-rs/cargo@v1
+ with:
+ command: clippy
+ args: --workspace --tests --examples -- -D warnings

.gitignore

@@ -0,0 +1,36 @@
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+data/
+dist/
+target/
+
+# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries
+# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+*tokenizer*.json
+*.npz
+
+perf.data
+flamegraph.svg
+*.dylib
+*.so
+*.swp
+trace-*.json
+
+candle-wasm-examples/*/build
+candle-wasm-examples/*/*.bin
+candle-wasm-examples/*/*.jpeg
+candle-wasm-examples/*/*.wav
+candle-wasm-examples/*/*.safetensors
+candle-wasm-examples/*/package-lock.json
+
+.DS_Store
+.idea/*

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "candle-examples/examples/flash-attn/cutlass"]
+ path = candle-flash-attn/cutlass
+ url = https://github.com/NVIDIA/cutlass.git

.pre-commit-config.yaml

@@ -0,0 +1,15 @@
+repos:
+ - repo: https://github.com/Narsil/pre-commit-rust
+ rev: 2eed6366172ef2a5186e8785ec0e67243d7d73d0
+ hooks:
+ - id: fmt
+ name: "Rust (fmt)"
+ - id: clippy
+ name: "Rust (clippy)"
+ args:
+ [
+ "--tests",
+ "--examples",
+ "--",
+ "-Dwarnings",
+ ]