Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fix Half-Quadratic Quantization and Dequantization on CPU #873

Open
wants to merge 6 commits into
base: master
Choose a base branch
from
+75 −8
Open

fix Half-Quadratic Quantization and Dequantization on CPU #873

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
95b2add
test_bitpack cpu/cuda
haricot Oct 21, 2024
0c76982
add test_bitpack 8 bit quantization cpu/cuda
haricot Oct 22, 2024
b162370
fix unnecessary nested cfg attributes
haricot Oct 22, 2024
62b3ae2
fix alloc/init cpu dequantize hqq
haricot Oct 22, 2024
f816446
ensuring contiguous data slices
haricot Oct 22, 2024
c5e7518
Revert "ensuring contiguous data slices to see result in CI"
haricot Oct 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions mistralrs-quant/src/hqq/hqq_cpu.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,8 +65,9 @@ pub(crate) struct Dequant4Bit {
}

impl Dequant4Bit {
fn dequantize<T: WithDType>(&self, w: &[u8], s: &[T], z: &[T]) -> Vec<T> {
let mut out = Vec::with_capacity(w.len());
fn dequantize<T: WithDType + Default>(&self, w: &[u8], s: &[T], z: &[T]) -> Vec<T> {
let output_size = w.len() * 2;
let mut out = vec![T::default(); output_size];
for (i, w) in w.iter().enumerate() {
let j = i % self.w;
let nrows = self.h * self.w;
Expand Down Expand Up @@ -125,8 +126,9 @@ pub(crate) struct Dequant2Bit {
}

impl Dequant2Bit {
fn dequantize<T: WithDType>(&self, w: &[u8], s: &[T], z: &[T]) -> Vec<T> {
let mut out = Vec::with_capacity(w.len());
fn dequantize<T: WithDType + Default>(&self, w: &[u8], s: &[T], z: &[T]) -> Vec<T> {
let output_size = w.len() * 4;
let mut out = vec![T::default(); output_size];
for (i, w) in w.iter().enumerate() {
let j = i % self.w;
let nrows = self.h * self.w;
Expand Down Expand Up @@ -187,8 +189,9 @@ pub(crate) struct Dequant1Bit {
}

impl Dequant1Bit {
fn dequantize<T: WithDType>(&self, w: &[u8], s: &[T], z: &[T]) -> Vec<T> {
let mut out = Vec::with_capacity(w.len());
fn dequantize<T: WithDType + Default>(&self, w: &[u8], s: &[T], z: &[T]) -> Vec<T> {
let output_size = w.len() * 8;
let mut out = vec![T::default(); output_size];
for (i, w) in w.iter().enumerate() {
let j = i % self.w;
let nrows = self.h * self.w;
Expand Down Expand Up @@ -253,8 +256,9 @@ pub(crate) struct Dequant3Bit {
}

impl Dequant3Bit {
fn dequantize<T: WithDType>(&self, w: &[i32], s: &[T], z: &[T]) -> Vec<T> {
let mut out = Vec::with_capacity(w.len());
fn dequantize<T: WithDType + Default>(&self, w: &[i32], s: &[T], z: &[T]) -> Vec<T> {
let output_size = w.len() * 10;
let mut out = vec![T::default(); output_size];
for (i, w) in w.iter().enumerate() {
let j = i % self.w;
let nrows = self.h * self.w;
Expand Down
63 changes: 63 additions & 0 deletions mistralrs-quant/src/utils/ops.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -398,4 +398,67 @@ mod tests {
assert_eq!(bv, [0b00001111]);
assert_eq!(c, [0b11111111]);
}

#[cfg(not(feature = "cuda"))]
#[test]
fn test_bitpack_8bit() {
use crate::HqqBits;
use candle_core::{Device, Tensor};
let bits = HqqBits::Eight;
let device = &Device::Cpu;
let wq = Tensor::from_vec(vec![257_i32, 258, 259, 260, 511, 512], (3, 2), &device).unwrap();
let c = bits.bitpack_type()(wq.clone())
.unwrap()
.to_vec2::<u8>()
.unwrap();
assert_eq!(c, [[1, 2], [3, 4], [255, 0]]);
}

#[cfg(feature = "cuda")]
#[test]
fn test_bitpack_8bit() {
use crate::HqqBits;
use candle_core::DType;
use candle_core::{Device, Tensor};
let bits = HqqBits::Eight;
let device = Device::new_cuda(0).unwrap();
let wq = Tensor::from_vec(vec![257_i32, 258, 259, 260, 511, 512], (3, 2), &device).unwrap();
let c = bits.bitpack_type()(wq.clone())
.unwrap()
.to_dtype(DType::U8)
.unwrap()
.to_vec2::<u8>()
.unwrap();
assert_eq!(c, [[1, 2], [3, 4], [255, 0]]);
}

#[cfg(not(feature = "cuda"))]
#[test]
fn test_bitpack_4bit() {
use crate::HqqBits;
use candle_core::{Device, Tensor};
let bits = HqqBits::Four;
let device = &Device::Cpu;
let wq = Tensor::from_vec(vec![1_u8, 2, 3, 4, 5, 6], (3, 2), &device).unwrap();
let c = bits.bitpack_type()(wq.clone())
.unwrap()
.to_vec2::<u8>()
.unwrap();
assert_eq!(c, [[19, 36]]);
}

#[cfg(feature = "cuda")]
#[test]
fn test_bitpack_4bit() {
use crate::HqqBits;
use candle_core::{Device, Tensor};
let bits = HqqBits::Four;
let device = Device::new_cuda(0).unwrap();
let wq = Tensor::from_vec(vec![1_u8, 2, 3, 4, 5, 6], (3, 2), &device).unwrap();
let c = bits.bitpack_type()(wq.clone())
.unwrap()
.to_vec2::<u8>()
.unwrap();
assert_eq!(c, [[19, 36]]);
}
}