Bump to version 0.2.2 (#619)

* Bump to version 0.2.2 * Update output * Format
EricLBuehler · Jul 24, 2024 · 3262fda · 3262fda
1 parent 7acdd1c
commit 3262fda
Show file tree

Hide file tree

Showing 12 changed files with 88 additions and 84 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -13,7 +13,7 @@ exclude = [
 resolver = "2"
 
 [workspace.package]
-version = "0.2.1"
+version = "0.2.2"
 edition = "2021"
 description = "Fast and easy LLM serving."
 homepage = "https://github.com/EricLBuehler/mistral.rs"

diff --git a/README.md b/README.md
@@ -35,10 +35,10 @@ Please submit requests for new models [here](https://github.com/EricLBuehler/mis
  ./mistralrs_server -i toml -f toml-selectors/anymoe_lora.toml
  ```
 
-- 💎 Run the Gemma 2 model
+- 🦙 Run the Llama 3.1 model
 
  ```
- ./mistralrs_server -i plain -m google/gemma-2-9b-it -a gemma2
+ ./mistralrs_server -i plain -m meta-llama/Meta-Llama-3.1-8B-Instruct -a llama
  ```
 
 - φ³ Run the Phi 3 model with 128K context window
@@ -189,7 +189,7 @@ Please submit more benchmarks via raising an issue!
 > Note: You can use our [Docker containers here](https://github.com/EricLBuehler/mistral.rs/pkgs/container/mistral.rs).
 > Learn more about running Docker containers: https://docs.docker.com/engine/reference/run/
 
-> Note: You can use pre-built `mistralrs-server` binaries [here](https://github.com/EricLBuehler/mistral.rs/releases/tag/v0.2.1)
+> Note: You can use pre-built `mistralrs-server` binaries [here](https://github.com/EricLBuehler/mistral.rs/releases/tag/v0.2.2)
 
 - Install the [Python package here](mistralrs-pyo3/README.md).
 

diff --git a/mistralrs-bench/Cargo.toml b/mistralrs-bench/Cargo.toml
@@ -18,7 +18,7 @@ candle-core.workspace = true
 serde.workspace = true
 serde_json.workspace = true
 clap.workspace = true
-mistralrs-core = { version = "0.2.1", path = "../mistralrs-core" }
+mistralrs-core = { version = "0.2.2", path = "../mistralrs-core" }
 tracing.workspace = true
 either.workspace = true
 tokio.workspace = true

diff --git a/mistralrs-core/Cargo.toml b/mistralrs-core/Cargo.toml
@@ -64,13 +64,13 @@ tracing-subscriber.workspace = true
 derive-new = "0.6.0"
 itertools = "0.13.0"
 sysinfo = "0.30.12"
-mistralrs-vision = { version = "0.2.1", path = "../mistralrs-vision" }
+mistralrs-vision = { version = "0.2.2", path = "../mistralrs-vision" }
 csv = "1.3.0"
 reqwest.workspace = true
 base64.workspace = true
 bytemuck_derive = "1.7.0"
 plotly = { version = "0.9.0", features = ["kaleido"], optional = true }
-mistralrs-paged-attn = { version = "0.2.1", path = "../mistralrs-paged-attn", optional = true }
+mistralrs-paged-attn = { version = "0.2.2", path = "../mistralrs-paged-attn", optional = true }
 
 [features]
 default = ["plotly"]

diff --git a/mistralrs-core/src/pipeline/isq.rs b/mistralrs-core/src/pipeline/isq.rs
@@ -142,74 +142,78 @@ pub trait IsqModel {
  {
  let (tensors, mapper) = self.get_biases();
  let total_tensors = tensors.len();
- info!("Applying in-situ quantization bias device mapping to {total_tensors} biases.");
- let bar = ProgressBar::new(total_tensors as u64);
- bar.set_style(
- ProgressStyle::default_bar()
- .template("[{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta})")
- .unwrap()
- .progress_chars("#>-"),
- );
-
- let mut devices = Vec::new();
- for (_, layer) in &tensors {
- let device = if let Some(layer) = layer {
- mapper.device_for(*layer, false).unwrap_or(&device)
- } else {
- &device
- };
- devices.push(device.clone());
- }
-
- let t_start = Instant::now();
- #[cfg(not(feature = "metal"))]
- {
- // NOTE(EricLBuehler): On version 0.2.0, remove this
- let isq_low_mem = std::env::var("ISQ_LOW_MEMORY").is_ok();
- if isq_low_mem {
- warn!("ISQ_LOW_MEMORY is set but as of version 0.1.24, this is irrelevant");
+ if total_tensors > 0 {
+ info!(
+ "Applying in-situ quantization bias device mapping to {total_tensors} biases."
+ );
+ let bar = ProgressBar::new(total_tensors as u64);
+ bar.set_style(
+ ProgressStyle::default_bar()
+ .template("[{elapsed_precise}] [{bar:40.cyan/blue}] {pos}/{len} ({eta})")
+ .unwrap()
+ .progress_chars("#>-"),
+ );
+
+ let mut devices = Vec::new();
+ for (_, layer) in &tensors {
+ let device = if let Some(layer) = layer {
+ mapper.device_for(*layer, false).unwrap_or(&device)
+ } else {
+ &device
+ };
+ devices.push(device.clone());
  }
 
- info!("Applying ISQ on {} threads.", rayon::current_num_threads());
-
- use indicatif::ParallelProgressIterator;
- use rayon::iter::{
- IndexedParallelIterator, IntoParallelIterator, ParallelIterator,
- };
- tensors
- .into_par_iter()
- .zip(devices)
- .progress_with(bar)
- .for_each(|((tensor, _), device)| {
- if let Some(tensor) = tensor {
- *tensor = tensor
- .to_device(&device)
- .unwrap()
- .to_dtype(DType::F32)
- .unwrap();
- }
- });
- }
+ let t_start = Instant::now();
+ #[cfg(not(feature = "metal"))]
+ {
+ // NOTE(EricLBuehler): On version 0.2.0, remove this
+ let isq_low_mem = std::env::var("ISQ_LOW_MEMORY").is_ok();
+ if isq_low_mem {
+ warn!("ISQ_LOW_MEMORY is set but as of version 0.1.24, this is irrelevant");
+ }
+
+ info!("Applying ISQ on {} threads.", rayon::current_num_threads());
+
+ use indicatif::ParallelProgressIterator;
+ use rayon::iter::{
+ IndexedParallelIterator, IntoParallelIterator, ParallelIterator,
+ };
+ tensors
+ .into_par_iter()
+ .zip(devices)
+ .progress_with(bar)
+ .for_each(|((tensor, _), device)| {
+ if let Some(tensor) = tensor {
+ *tensor = tensor
+ .to_device(&device)
+ .unwrap()
+ .to_dtype(DType::F32)
+ .unwrap();
+ }
+ });
+ }
 
- #[cfg(feature = "metal")]
- {
- use indicatif::ProgressIterator;
- tensors
- .into_iter()
- .zip(devices)
- .progress_with(bar)
- .for_each(|((tensor, _), device)| {
- if let Some(tensor) = tensor {
- *tensor = tensor
- .to_device(&device)
- .unwrap()
- .to_dtype(DType::F32)
- .unwrap();
- }
- });
+ #[cfg(feature = "metal")]
+ {
+ use indicatif::ProgressIterator;
+ tensors
+ .into_iter()
+ .zip(devices)
+ .progress_with(bar)
+ .for_each(|((tensor, _), device)| {
+ if let Some(tensor) = tensor {
+ *tensor = tensor
+ .to_device(&device)
+ .unwrap()
+ .to_dtype(DType::F32)
+ .unwrap();
+ }
+ });
+ }
+ let delta = Instant::now().duration_since(t_start).as_secs_f32();
+ info!("Applied in-situ quantization device mapping. Took {delta:.2}s",);
  }
- let delta = Instant::now().duration_since(t_start).as_secs_f32();
- info!("Applied in-situ quantization device mapping. Took {delta:.2}s",);
  }
  Ok(())
  }

diff --git a/mistralrs-pyo3/Cargo.toml b/mistralrs-pyo3/Cargo.toml
@@ -17,7 +17,7 @@ doc = false
 
 [dependencies]
 pyo3.workspace = true
-mistralrs-core = { version = "0.2.1", path = "../mistralrs-core", features = ["pyo3_macros"] }
+mistralrs-core = { version = "0.2.2", path = "../mistralrs-core", features = ["pyo3_macros"] }
 serde.workspace = true
 serde_json.workspace = true
 candle-core.workspace = true

diff --git a/mistralrs-pyo3/Cargo_template.toml b/mistralrs-pyo3/Cargo_template.toml
@@ -17,7 +17,7 @@ doc = false
 
 [dependencies]
 pyo3.workspace = true
-mistralrs-core = { version = "0.2.1", path = "../mistralrs-core", features=["pyo3_macros","$feature_name"] }
+mistralrs-core = { version = "0.2.2", path = "../mistralrs-core", features=["pyo3_macros","$feature_name"] }
 serde.workspace = true
 serde_json.workspace = true
 candle-core = { git = "https://github.com/EricLBuehler/candle.git", version = "0.6.0", rev = "c967be9", features=["$feature_name"] }

diff --git a/mistralrs-pyo3/pyproject.toml b/mistralrs-pyo3/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "mistralrs"
-version = "0.2.1"
+version = "0.2.2"
 requires-python = ">=3.8"
 classifiers = [
  "Programming Language :: Rust",

diff --git a/mistralrs-pyo3/pyproject_template.toml b/mistralrs-pyo3/pyproject_template.toml
@@ -4,7 +4,7 @@ build-backend = "maturin"
 
 [project]
 name = "$name"
-version = "0.2.1"
+version = "0.2.2"
 requires-python = ">=3.8"
 classifiers = [
  "Programming Language :: Rust",

diff --git a/mistralrs-server/Cargo.toml b/mistralrs-server/Cargo.toml
@@ -22,7 +22,7 @@ axum = { version = "0.7.4", features = ["tokio"] }
 tower-http = { version = "0.5.1", features = ["cors"]}
 utoipa = { version = "4.2", features = ["axum_extras"] }
 utoipa-swagger-ui = { version = "7.1.0", features = ["axum"]}
-mistralrs-core = { version = "0.2.1", path = "../mistralrs-core" }
+mistralrs-core = { version = "0.2.2", path = "../mistralrs-core" }
 indexmap.workspace = true
 accelerate-src = { workspace = true, optional = true }
 intel-mkl-src = { workspace = true, optional = true }

diff --git a/mistralrs/Cargo.toml b/mistralrs/Cargo.toml
@@ -12,7 +12,7 @@ license.workspace = true
 homepage.workspace = true
 
 [dependencies]
-mistralrs-core = { version = "0.2.1", path = "../mistralrs-core" }
+mistralrs-core = { version = "0.2.2", path = "../mistralrs-core" }
 anyhow.workspace = true
 tokio.workspace = true
 candle-core.workspace = true