From 5c15e8c947720bddccd8088e61f18c6966ace1df Mon Sep 17 00:00:00 2001 From: Isamu Isozaki Date: Thu, 14 Mar 2024 21:38:07 +0900 Subject: [PATCH] Pass `'model_kwargs` for `outlines.models.llamacpp` as dict (#744) Fixes [this issue](https://github.com/outlines-dev/outlines/issues/743) --- outlines/models/llamacpp.py | 1 + 1 file changed, 1 insertion(+) diff --git a/outlines/models/llamacpp.py b/outlines/models/llamacpp.py index 010aaafbf..40adc05b2 100644 --- a/outlines/models/llamacpp.py +++ b/outlines/models/llamacpp.py @@ -25,6 +25,7 @@ def __init__(self, model: "Llama"): def llamacpp(model_path: str, device: Optional[str] = None, **model_kwargs) -> LlamaCpp: from llama_cpp import Llama + model_kwargs = model_kwargs.get("model_kwargs", {}) if device == "cuda": model_kwargs["n_gpu_layers"].setdefault(-1)