From ac2ab9e9d2b1a2d3d82003278df8c73f246d11f0 Mon Sep 17 00:00:00 2001 From: Eric Buehler Date: Tue, 15 Oct 2024 20:50:41 -0400 Subject: [PATCH] Add Phi 3.5 moe support for uqff --- mistralrs-core/src/models/phi3_5_moe.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mistralrs-core/src/models/phi3_5_moe.rs b/mistralrs-core/src/models/phi3_5_moe.rs index 0f02c1c68..8a4211e14 100644 --- a/mistralrs-core/src/models/phi3_5_moe.rs +++ b/mistralrs-core/src/models/phi3_5_moe.rs @@ -749,6 +749,7 @@ impl IsqModel for Model { uvb_l .pp("post_attention_layernorm") .add(&layer.post_attention_layernorm); + uvb_l.pp("block_sparse_moe").pp("gate").add(&layer.mlp.gate); } uvb.to_safetensors() @@ -767,6 +768,7 @@ impl IsqModel for Model { uvb_l .pp("post_attention_layernorm") .add(&layer.post_attention_layernorm); + uvb_l.pp("block_sparse_moe").pp("gate").add(&layer.mlp.gate); let uvb_attn = uvb_l.pp("self_attn"); uvb_attn.pp("q_proj").add(&layer.self_attn.q_proj);