Feat/fireworks integration

Dockerfile.fireworks

@@ -0,0 +1,54 @@
+FROM python:3.11.6-slim-bookworm as base
+
+# Install poetry
+RUN pip install pipx
+RUN python3 -m pipx ensurepath
+RUN pipx install poetry==1.8.3
+ENV PATH="/root/.local/bin:$PATH"
+ENV PATH=".venv/bin/:$PATH"
+
+RUN apt update && apt install -y \
+  build-essential
+
+# https://python-poetry.org/docs/configuration/#virtualenvsin-project
+ENV POETRY_VIRTUALENVS_IN_PROJECT=true
+
+FROM base as dependencies
+WORKDIR /home/worker/app
+COPY pyproject.toml poetry.lock ./
+
+ARG POETRY_EXTRAS="ui llms-fireworks embeddings-fireworks vector-stores-qdrant embeddings-openai"
+RUN poetry install --no-root --extras "${POETRY_EXTRAS}"
+
+FROM base as app
+ENV PYTHONUNBUFFERED=1
+ENV PORT=8080
+ENV APP_ENV=prod
+ENV PYTHONPATH="$PYTHONPATH:/home/worker/app/private_gpt/"
+EXPOSE 8080
+
+# Prepare a non-root user
+# More info about how to configure UIDs and GIDs in Docker:
+# https://github.com/systemd/systemd/blob/main/docs/UIDS-GIDS.md
+
+# Define the User ID (UID) for the non-root user
+# UID 100 is chosen to avoid conflicts with existing system users
+ARG UID=100
+
+# Define the Group ID (GID) for the non-root user
+# GID 65534 is often used for the 'nogroup' or 'nobody' group
+ARG GID=65534
+
+RUN adduser --system --gid ${GID} --uid ${UID} --home /home/worker worker
+WORKDIR /home/worker/app
+
+RUN chown worker /home/worker/app
+RUN mkdir local_data && chown worker local_data
+RUN mkdir models && chown worker models
+COPY --chown=worker --from=dependencies /home/worker/app/.venv/ .venv
+COPY --chown=worker private_gpt/ private_gpt
+COPY --chown=worker *.yaml .
+COPY --chown=worker scripts/ scripts
+
+USER worker
+ENTRYPOINT python -m private_gpt

docker-compose.yaml

@@ -1,13 +1,12 @@
 services:
-
   #-----------------------------------
   #---- Private-GPT services ---------
   #-----------------------------------
 
   # Private-GPT service for the Ollama CPU and GPU modes
   # This service builds from an external Dockerfile and runs the Ollama mode.
   private-gpt-ollama:
-    image: ${PGPT_IMAGE:-zylonai/private-gpt}:${PGPT_TAG:-0.6.2}-ollama  # x-release-please-version
+    image: ${PGPT_IMAGE:-zylonai/private-gpt}:${PGPT_TAG:-0.6.2}-ollama # x-release-please-version
     build:
       context: .
       dockerfile: Dockerfile.ollama
@@ -80,7 +79,7 @@ services:
   ollama-cpu:
     image: ollama/ollama:latest
     volumes:
-      - ./models:/root/.ollama
+      - ./local_data:/root/.ollama
     profiles:
       - ""
       - ollama-cpu
@@ -98,4 +97,22 @@ services:
               count: 1
               capabilities: [gpu]
     profiles:
-      - ollama-cuda
+      - ollama-cuda
+
+    # fireworks service
+  private-gpt-fireworks:
+    build:
+      context: .
+      dockerfile: Dockerfile.fireworks
+    volumes:
+      - ./local_data/:/home/worker/app/local_data
+    ports:
+      - "3001:8080"
+    environment:
+      PORT: 8080
+      PGPT_PROFILES: fireworks
+      FIREWORKS_API_KEY: ${FIREWORKS_API_KEY}
+    env_file:
+      - .env
+    profiles:
+      - fireworks

private_gpt/components/embedding/embedding_component.py

@@ -67,6 +67,19 @@ def __init__(self, settings: Settings) -> None:
                     api_key=api_key,
                     model=model,
                 )
+            case "fireworks":
+                try:
+                    from llama_index.embeddings.fireworks import FireworksEmbedding
+                except ImportError as e:
+                    raise ImportError(
+                        "FireworksEmbedding dependencies not found, install with `poetry install --extras embeddings-fireworks`"
+                    ) from e
+
+                api_key = settings.fireworks.embedding_api_key or settings.fireworks.api_key
+
+                self.embedding_model = FireworksEmbedding(
+                    api_key=api_key,
+                )
             case "ollama":
                 try:
                     from llama_index.embeddings.ollama import (  # type: ignore

private_gpt/components/llm/llm_component.py

@@ -102,6 +102,19 @@ def __init__(self, settings: Settings) -> None:
                     api_key=openai_settings.api_key,
                     model=openai_settings.model,
                 )
+            case "fireworks":
+                try:
+                    from llama_index.llms.fireworks import Fireworks  # type: ignore
+                except ImportError as e:
+                    raise ImportError(
+                        "fireworks dependencies not found, install with `poetry install --extras llms-fireworks`"
+                    ) from e
+
+                fireworks_settings = settings.fireworks
+                self.llm = Fireworks(
+                    model=fireworks_settings.model,
+                    api_key=fireworks_settings.api_key,
+                )
             case "openailike":
                 try:
                     from llama_index.llms.openai_like import OpenAILike  # type: ignore

private_gpt/settings/settings.py

@@ -115,6 +115,7 @@ class LLMSettings(BaseModel):
         "mock",
         "ollama",
         "gemini",
+        "fireworks",
     ]
     max_new_tokens: int = Field(
         256,
@@ -197,7 +198,7 @@ class HuggingFaceSettings(BaseModel):
 
 class EmbeddingSettings(BaseModel):
     mode: Literal[
-        "huggingface", "openai", "azopenai", "sagemaker", "ollama", "mock", "gemini"
+        "huggingface", "openai", "azopenai", "sagemaker", "ollama", "mock", "gemini","fireworks"
     ]
     ingest_mode: Literal["simple", "batch", "parallel", "pipeline"] = Field(
         "simple",
@@ -260,6 +261,21 @@ class OpenAISettings(BaseModel):
         description="OpenAI embedding Model to use. Example: 'text-embedding-3-large'.",
     )
 
+class FireWorksSettings(BaseModel):
+    api_key: str
+    model: str = Field(
+        "accounts/fireworks/models/llama-v3p1-70b-instruct",
+        description="FireWorks Model to use. Example: 'accounts/fireworks/models/llama-v3p1-70b-instruct'.",
+    )
+    # embedding_api_base: str = Field(
+    #     None,
+    #     description="Base URL of OpenAI API. Example: 'https://api.openai.com/v1'.",
+    # )
+    embedding_api_key: str
+    # embedding_model: str = Field(
+    #     "text-embedding-ada-002",
+    #     description="OpenAI embedding Model to use. Example: 'text-embedding-3-large'.",
+    # )
 
 class GeminiSettings(BaseModel):
     api_key: str
@@ -586,6 +602,7 @@ class Settings(BaseModel):
     huggingface: HuggingFaceSettings
     sagemaker: SagemakerSettings
     openai: OpenAISettings
+    fireworks: FireWorksSettings
     gemini: GeminiSettings
     ollama: OllamaSettings
     azopenai: AzureOpenAISettings

private_gpt/ui/ui.py

@@ -377,7 +377,7 @@ def _build_ui_blocks(self) -> gr.Blocks:
             ".contain { display: flex !important; flex-direction: column !important; }"
             "#component-0, #component-3, #component-10, #component-8  { height: 100% !important; }"
             "#chatbot { flex-grow: 1 !important; overflow: auto !important;}"
-            "#col { height: calc(100vh - 112px - 16px) !important; }"
+            "#col { min-height: calc(100vh - 112px - 16px) !important; }"
             "hr { margin-top: 1em; margin-bottom: 1em; border: 0; border-top: 1px solid #FFF; }"
             ".avatar-image { background-color: antiquewhite; border-radius: 2px; }"
             ".footer { text-align: center; margin-top: 20px; font-size: 14px; display: flex; align-items: center; justify-content: center; }"
@@ -518,6 +518,7 @@ def get_model_label() -> str | None:
                         model_mapping = {
                             "llamacpp": config_settings.llamacpp.llm_hf_model_file,
                             "openai": config_settings.openai.model,
+                            "fireworks": config_settings.fireworks.model,
                             "openailike": config_settings.openai.model,
                             "azopenai": config_settings.azopenai.llm_model,
                             "sagemaker": config_settings.sagemaker.llm_endpoint_name,

pyproject.toml

@@ -37,6 +37,9 @@ llama-index-vector-stores-postgres = {version ="^0.1.11", optional = true}
 llama-index-vector-stores-clickhouse = {version ="^0.1.3", optional = true}
 llama-index-storage-docstore-postgres = {version ="^0.1.3", optional = true}
 llama-index-storage-index-store-postgres = {version ="^0.1.4", optional = true}
+llama-index-llms-fireworks = {version = "^0.1.5", optional = true}
+llama-index-embeddings-fireworks = {version = "^0.1.2", optional = true}
+
 # Postgres
 psycopg2-binary = {version ="^2.9.9", optional = true}
 asyncpg = {version="^0.29.0", optional = true}
@@ -90,6 +93,8 @@ vector-stores-postgres = ["llama-index-vector-stores-postgres"]
 vector-stores-milvus = ["llama-index-vector-stores-milvus"]
 storage-nodestore-postgres = ["llama-index-storage-docstore-postgres","llama-index-storage-index-store-postgres","psycopg2-binary","asyncpg"]
 rerank-sentence-transformers = ["torch", "sentence-transformers"]
+llms-fireworks = ["llama-index-llms-fireworks"]
+embeddings-fireworks = ["llama-index-embeddings-fireworks"]
 
 [tool.poetry.group.dev.dependencies]
 black = "^22"

settings-fireworks.yaml

@@ -0,0 +1,13 @@
+server:
+  env_name: ${APP_ENV:fireworks}
+
+llm:
+  mode: fireworks
+
+embedding:
+  mode: fireworks
+
+fireworks:
+  api_key: ${FIREWORKS_API_KEY:}
+  model: "accounts/fireworks/models/llama-v3p1-70b-instruct"
+  #poetry install --extras "ui llms-fireworks embeddings-fireworks vector-stores-qdrant embeddings-openai"

settings.yaml

@@ -52,7 +52,7 @@ llm:
   context_window: 3900
   # Select your tokenizer. Llama-index tokenizer is the default.
   # tokenizer: meta-llama/Meta-Llama-3.1-8B-Instruct
-  temperature: 0.1      # The temperature of the model. Increasing the temperature will make the model answer more creatively. A value of 0.1 would be more factual. (Default: 0.1)
+  temperature: 0.1 # The temperature of the model. Increasing the temperature will make the model answer more creatively. A value of 0.1 would be more factual. (Default: 0.1)
 
 rag:
   similarity_top_k: 2
@@ -68,19 +68,19 @@ summarize:
   use_async: true
 
 clickhouse:
-    host: localhost
-    port: 8443
-    username: admin
-    password: clickhouse
-    database: embeddings
+  host: localhost
+  port: 8443
+  username: admin
+  password: clickhouse
+  database: embeddings
 
 llamacpp:
   llm_hf_repo_id: lmstudio-community/Meta-Llama-3.1-8B-Instruct-GGUF
   llm_hf_model_file: Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf
-  tfs_z: 1.0            # Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting
-  top_k: 40             # Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. (Default: 40)
-  top_p: 1.0            # Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.9)
-  repeat_penalty: 1.1   # Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. (Default: 1.1)
+  tfs_z: 1.0 # Tail free sampling is used to reduce the impact of less probable tokens from the output. A higher value (e.g., 2.0) will reduce the impact more, while a value of 1.0 disables this setting
+  top_k: 40 # Reduces the probability of generating nonsense. A higher value (e.g. 100) will give more diverse answers, while a lower value (e.g. 10) will be more conservative. (Default: 40)
+  top_p: 1.0 # Works together with top-k. A higher value (e.g., 0.95) will lead to more diverse text, while a lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.9)
+  repeat_penalty: 1.1 # Sets how strongly to penalize repetitions. A higher value (e.g., 1.5) will penalize repetitions more strongly, while a lower value (e.g., 0.9) will be more lenient. (Default: 1.1)
 
 embedding:
   # Should be matching the value above in most cases
@@ -126,11 +126,16 @@ openai:
   model: gpt-3.5-turbo
   embedding_api_key: ${OPENAI_API_KEY:}
 
+fireworks:
+  api_key: ${FIREWORKS_API_KEY:}
+  model: "accounts/fireworks/models/llama-v3p1-70b-instruct"
+  embedding_api_key: ${FIREWORKS_API_KEY:}
+
 ollama:
   llm_model: llama3.1
   embedding_model: nomic-embed-text
   api_base: http://localhost:11434
-  embedding_api_base: http://localhost:11434  # change if your embedding model runs on another ollama
+  embedding_api_base: http://localhost:11434 # change if your embedding model runs on another ollama
   keep_alive: 5m
   request_timeout: 120.0
   autopull_models: true