Fix the fast bf16 gemm cublas kernels. (#2274)

* Use flash-attn in gemma.

* Fix for the fast bf16 cublas gemm.

* Fix some clippy lints.

* Fix another lint.

* Proper clippy fix.

candle-core/examples/cuda_basics.rs

@@ -9,8 +9,10 @@ use candle_core::{Device, Tensor};
 
 fn main() -> Result<()> {
  let device = Device::new_cuda(0)?;
- let x = Tensor::randn(0f32, 1.0, (8 * 4096, 8 * 4096), &device)?;
+ let x = Tensor::randn(0f32, 1.0, (8 * 4096, 8 * 4096), &device)?
+ .to_dtype(candle_core::DType::BF16)?;
  candle_core::cuda::set_gemm_reduced_precision_f32(false);
+ candle_core::cuda::set_gemm_reduced_precision_bf16(false);
  let _x1 = x.matmul(&x)?;
  drop(_x1);
  let start_time = std::time::Instant::now();
@@ -19,6 +21,7 @@ fn main() -> Result<()> {
  println!("fp32: {:?}", start_time.elapsed());
  drop(_x1);
  candle_core::cuda::set_gemm_reduced_precision_f32(true);
+ candle_core::cuda::set_gemm_reduced_precision_bf16(true);
  let _x1 = x.matmul(&x)?;
  drop(_x1);
  let start_time = std::time::Instant::now();

candle-core/src/cpu_backend/mod.rs

@@ -121,7 +121,8 @@ impl ReduceIndex {
  let dst_len = src_l.shape().elem_count() / reduce_dim_size;
  let mut dst: Vec<U> = Vec::with_capacity(dst_len);
  let dst_to_set = dst.spare_capacity_mut();
- let dst_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(dst_to_set) };
+ let dst_to_set =
+ unsafe { std::mem::transmute::<&mut [std::mem::MaybeUninit<U>], &mut [U]>(dst_to_set) };
  match src_l.contiguous_offsets() {
  Some((o1, o2)) => {
  let src = &src[o1..o2];

candle-core/src/cpu_backend/utils.rs

@@ -174,7 +174,9 @@ pub fn binary_map_vec<T: Copy, F: FnMut(T, T) -> T, FV: FnMut(&[T], &[T], &mut [
  (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => {
  let mut ys: Vec<T> = Vec::with_capacity(el_count);
  let ys_to_set = ys.spare_capacity_mut();
- let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) };
+ let ys_to_set = unsafe {
+ std::mem::transmute::<&mut [std::mem::MaybeUninit<T>], &mut [T]>(ys_to_set)
+ };
  f_vec(&lhs[o_l1..o_l2], &rhs[o_r1..o_r2], ys_to_set);
  // SAFETY: values are all set by f_vec.
  unsafe { ys.set_len(el_count) };
@@ -185,7 +187,9 @@ pub fn binary_map_vec<T: Copy, F: FnMut(T, T) -> T, FV: FnMut(&[T], &[T], &mut [
  let rhs = &rhs[ob.start..ob.start + ob.len];
  let mut ys: Vec<T> = Vec::with_capacity(el_count);
  let ys_to_set = ys.spare_capacity_mut();
- let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) };
+ let ys_to_set = unsafe {
+ std::mem::transmute::<&mut [std::mem::MaybeUninit<T>], &mut [T]>(ys_to_set)
+ };
  let mut dst_i = 0;
  for src_i in (o_l1..o_l2).step_by(ob.len) {
  f_vec(
@@ -224,7 +228,9 @@ pub fn binary_map_vec<T: Copy, F: FnMut(T, T) -> T, FV: FnMut(&[T], &[T], &mut [
  let lhs = &lhs[ob.start..ob.start + ob.len];
  let mut ys: Vec<T> = Vec::with_capacity(el_count);
  let ys_to_set = ys.spare_capacity_mut();
- let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) };
+ let ys_to_set = unsafe {
+ std::mem::transmute::<&mut [std::mem::MaybeUninit<T>], &mut [T]>(ys_to_set)
+ };
  let mut dst_i = 0;
  for src_i in (o_r1..o_r2).step_by(ob.len) {
  f_vec(
@@ -311,7 +317,9 @@ pub fn unary_map_vec<T: Copy, U: Copy, F: FnMut(T) -> U, FV: FnMut(&[T], &mut [U
  crate::StridedBlocks::SingleBlock { start_offset, len } => {
  let mut ys: Vec<U> = Vec::with_capacity(len);
  let ys_to_set = ys.spare_capacity_mut();
- let ys_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(ys_to_set) };
+ let ys_to_set = unsafe {
+ std::mem::transmute::<&mut [std::mem::MaybeUninit<U>], &mut [U]>(ys_to_set)
+ };
  f_vec(&vs[start_offset..start_offset + len], ys_to_set);
  // SAFETY: values are all set by f_vec.
  unsafe { ys.set_len(len) };
@@ -333,7 +341,9 @@ pub fn unary_map_vec<T: Copy, U: Copy, F: FnMut(T) -> U, FV: FnMut(&[T], &mut [U
  } else {
  let mut ys: Vec<U> = Vec::with_capacity(el_count);
  let ys_to_set = ys.spare_capacity_mut();
- let ys_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(ys_to_set) };
+ let ys_to_set = unsafe {
+ std::mem::transmute::<&mut [std::mem::MaybeUninit<U>], &mut [U]>(ys_to_set)
+ };
  let mut dst_index = 0;
  for src_index in block_start_index {
  let vs = &vs[src_index..src_index + block_len];

candle-core/src/cuda_backend/mod.rs

@@ -2035,15 +2035,13 @@ unsafe fn gemm_strided_batched_bf16(
 
  let alpha_f32: f32 = cfg.gemm.alpha.to_f32();
  let beta_f32: f32 = cfg.gemm.beta.to_f32();
- let alpha = f16::from_f32(alpha_f32);
- let beta = f16::from_f32(beta_f32);
  // The type for alpha and beta depends on the computeType.
  // https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedex
  let (compute_type, alpha, beta) = if gemm_reduced_precision_bf16() {
  (
- sys::cublasComputeType_t::CUBLAS_COMPUTE_16F,
- (&alpha) as *const f16 as *const _,
- (&beta) as *const f16 as *const _,
+ sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_16BF,
+ (&alpha_f32) as *const f32 as *const _,
+ (&beta_f32) as *const f32 as *const _,
  )
  } else {
  (

candle-transformers/src/models/vgg.rs

@@ -54,8 +54,7 @@ impl ModuleT for Vgg<'_> {
 fn conv2d_block(convs: &[(usize, usize, &str)], vb: &VarBuilder) -> Result<FuncT<'static>> {
  let layers = convs
  .iter()
- .enumerate()
- .map(|(_, &(in_c, out_c, name))| {
+ .map(|&(in_c, out_c, name)| {
  candle_nn::conv2d(
  in_c,
  out_c,