common.py

import os
import argparse
import re

from dataclasses import dataclass, field
from typing import List

# Based on https://github.com/ggerganov/llama.cpp/blob/master/examples/common.cpp


@dataclass
class GptParams:
    seed: int = -1
    n_threads: int = min(4, os.cpu_count() or 1)
    n_predict: int = 128
    n_parts: int = -1
    n_ctx: int = 512
    n_batch: int = 8
    n_keep: int = 0

    ignore_eos: bool = False
    logit_bias: dict[int, float] = field(default_factory=dict)
    top_k: int = 40
    top_p: float = 0.95
    tfs_z: float = 1.00
    typical_p: float = 1.00
    temp: float = 0.80
    repeat_penalty: float = 1.10
    repeat_last_n: int = 64
    frequency_penalty: float = 0.0
    presence_penalty: float = 0.0
    mirostat: int = 0
    mirostat_tau: float = 5.0
    mirostat_eta: float = 0.1

    model: str = "./models/llama-7B/ggml-model.bin"
    prompt: str = ""
    path_session: str = ""
    input_prefix: str = " "
    input_suffix: str = ""
    antiprompt: List[str] = field(default_factory=list)

    lora_adapter: str = ""
    lora_base: str = ""

    memory_f16: bool = True
    random_prompt: bool = False
    use_color: bool = False
    interactive: bool = False

    embedding: bool = False
    interactive_start: bool = False

    instruct: bool = False
    penalize_nl: bool = True
    perplexity: bool = False
    use_mmap: bool = True
    use_mlock: bool = False
    mem_test: bool = False
    verbose_prompt: bool = False

    file: str = None

    # If chat ended prematurely, append this to the conversation to fix it.
    # Set to "\nUser:" etc.
    # This is an alternative to input_prefix which always adds it, so it potentially duplicates "User:""
    fix_prefix: str = ""
    input_echo: bool = (True,)

    # Default instructions for Alpaca
    # switch to "Human" and "Assistant" for Vicuna.
    # TODO: TBD how they are gonna handle this upstream
    instruct_inp_prefix: str = "\n\n### Instruction:\n\n"
    instruct_inp_suffix: str = "\n\n### Response:\n\n"


def gpt_params_parse(argv=None):
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter
    )
    parser.add_argument(
        "-s",
        "--seed",
        type=int,
        default=-1,
        help="RNG seed (use random seed for <= 0)",
        dest="seed",
    )
    parser.add_argument(
        "-t",
        "--threads",
        type=int,
        default=min(4, os.cpu_count() or 1),
        help="number of threads to use during computation",
        dest="n_threads",
    )
    parser.add_argument(
        "-n",
        "--n_predict",
        type=int,
        default=128,
        help="number of tokens to predict (-1 = infinity)",
        dest="n_predict",
    )
    parser.add_argument(
        "--n_parts", type=int, default=-1, help="number of model parts", dest="n_parts"
    )
    parser.add_argument(
        "-c",
        "--ctx_size",
        type=int,
        default=512,
        help="size of the prompt context",
        dest="n_ctx",
    )
    parser.add_argument(
        "-b",
        "--batch_size",
        type=int,
        default=8,
        help="batch size for prompt processing",
        dest="n_batch",
    )
    parser.add_argument(
        "--keep",
        type=int,
        default=0,
        help="number of tokens to keep from the initial prompt",
        dest="n_keep",
    )

    parser.add_argument(
        "-l",
        "--logit-bias",
        type=str,
        action="append",
        help="--logit-bias TOKEN_ID(+/-)BIAS",
        dest="logit_bias_str",
    )
    parser.add_argument(
        "--ignore-eos",
        action="store_true",
        help="ignore end of stream token and continue generating",
        dest="ignore_eos",
    )
    parser.add_argument(
        "--top_k", type=int, default=40, help="top-k sampling", dest="top_k"
    )
    parser.add_argument(
        "--top_p", type=float, default=0.95, help="top-p samplin", dest="top_p"
    )
    parser.add_argument(
        "--tfs",
        type=float,
        default=1.0,
        help="tail free sampling, parameter z (1.0 = disabled)",
        dest="tfs_z",
    )
    parser.add_argument(
        "--temp", type=float, default=0.80, help="temperature", dest="temp"
    )
    parser.add_argument(
        "--repeat_penalty",
        type=float,
        default=1.10,
        help="penalize repeat sequence of tokens",
        dest="repeat_penalty",
    )
    parser.add_argument(
        "--repeat_last_n",
        type=int,
        default=64,
        help="last n tokens to consider for penalize ",
        dest="repeat_last_n",
    )
    parser.add_argument(
        "--frequency_penalty",
        type=float,
        default=0.0,
        help="repeat alpha frequency penalty (0.0 = disabled)",
        dest="tfs_z",
    )
    parser.add_argument(
        "--presence_penalty",
        type=float,
        default=0.0,
        help="repeat alpha presence penalty (0.0 = disabled)",
        dest="presence_penalty",
    )
    parser.add_argument(
        "--mirostat",
        type=float,
        default=1.0,
        help="use Mirostat sampling.",
        dest="mirostat",
    )
    parser.add_argument(
        "--mirostat_ent",
        type=float,
        default=5.0,
        help="Mirostat target entropy, parameter tau represents the average surprise value",
        dest="mirostat_tau",
    )
    parser.add_argument(
        "--mirostat_lr",
        type=float,
        default=0.1,
        help="Mirostat learning rate, parameter eta",
        dest="mirostat_eta",
    )

    parser.add_argument(
        "-m",
        "--model",
        type=str,
        default="./models/llama-7B/ggml-model.bin",
        help="model path",
        dest="model",
    )
    parser.add_argument(
        "-p", "--prompt", type=str, default=None, help="initial prompt", dest="prompt"
    )
    parser.add_argument(
        "-f",
        "--file",
        type=str,
        default=None,
        help="file containing initial prompt to load",
        dest="file",
    )
    parser.add_argument(
        "--session",
        type=str,
        default=None,
        help="file to cache model state in (may be large!)",
        dest="path_session",
    )
    parser.add_argument(
        "--in-prefix",
        type=str,
        default="",
        help="string to prefix user inputs with",
        dest="input_prefix",
    )
    parser.add_argument(
        "--in-suffix", type=str, default="", help="append to input", dest="input_suffix"
    )
    parser.add_argument(
        "-r",
        "--reverse-prompt",
        type=str,
        action="append",
        help="poll user input upon seeing PROMPT (can be\nspecified more than once for multiple prompts).",
        dest="antiprompt",
    )

    parser.add_argument(
        "--lora",
        type=str,
        default="",
        help="apply LoRA adapter (implies --no-mmap)",
        dest="lora_adapter",
    )
    parser.add_argument(
        "--lora-base",
        type=str,
        default="",
        help="optional model to use as a base for the layers modified by the LoRA adapter",
        dest="lora_base",
    )

    parser.add_argument(
        "--memory_f32",
        action="store_false",
        help="use f32 instead of f16 for memory key+value",
        dest="memory_f16",
    )
    parser.add_argument(
        "--random-prompt",
        action="store_true",
        help="start with a randomized prompt.",
        dest="random_prompt",
    )
    parser.add_argument(
        "--color",
        action="store_true",
        help="colorise output to distinguish prompt and user input from generations",
        dest="use_color",
    )
    parser.add_argument(
        "-i",
        "--interactive",
        action="store_true",
        help="run in interactive mode",
        dest="interactive",
    )

    parser.add_argument("--embedding", action="store_true", help="", dest="embedding")
    parser.add_argument(
        "--interactive-first",
        action="store_true",
        help="run in interactive mode and wait for input right away",
        dest="interactive_start",
    )

    parser.add_argument(
        "-ins",
        "--instruct",
        action="store_true",
        help="run in instruction mode (use with Alpaca or Vicuna models)",
        dest="instruct",
    )
    parser.add_argument(
        "--no-penalize-nl",
        action="store_false",
        help="do not penalize newline token",
        dest="penalize_nl",
    )
    parser.add_argument(
        "--perplexity",
        action="store_true",
        help="compute perplexity over the prompt",
        dest="perplexity",
    )
    parser.add_argument(
        "--no-mmap",
        action="store_false",
        help="do not memory-map model (slower load but may reduce pageouts if not using mlock)",
        dest="use_mmap",
    )
    parser.add_argument(
        "--mlock",
        action="store_true",
        help="force system to keep model in RAM rather than swapping or compressing",
        dest="use_mlock",
    )
    parser.add_argument(
        "--mtest",
        action="store_true",
        help="compute maximum memory usage",
        dest="mem_test",
    )
    parser.add_argument(
        "--verbose-prompt",
        action="store_true",
        help="print prompt before generation",
        dest="verbose_prompt",
    )

    # Custom args
    parser.add_argument(
        "--fix-prefix",
        type=str,
        default="",
        help="append to input when generated n_predict tokens",
        dest="fix_prefix",
    )
    parser.add_argument(
        "--input-noecho",
        action="store_false",
        help="dont output the input",
        dest="input_echo",
    )

    parser.add_argument(
        "--interactive-start",
        action="store_true",
        help="run in interactive mode",
        dest="interactive",
    )

    args = parser.parse_args(argv)

    logit_bias_str = args.logit_bias_str
    delattr(args, "logit_bias_str")
    params = GptParams(**vars(args))

    if params.lora_adapter:
        params.use_mmap = False

    if logit_bias_str != None:
        for i in logit_bias_str:
            if m := re.match(r"(\d+)([-+]\d+)", i):
                params.logit_bias[int(m.group(1))] = float(m.group(2))

    return params


def gpt_random_prompt(rng):
    return [
        "So",
        "Once upon a time",
        "When",
        "The",
        "After",
        "If",
        "import",
        "He",
        "She",
        "They",
    ][rng % 10]


if __name__ == "__main__":
    print(gpt_params_parse())