docker-compose.yml

services:
  sealion-vllm:
    image: vllm/vllm-openai:v0.6.3
    container_name: sealion-vllm
    environment:
      # Disable the NVIDIA constraints
      # Remove this environment variable if the environment satisfies the constraints
      - NVIDIA_DISABLE_REQUIRE=true
    ports:
      - "8000:8000"
    restart: no
    volumes:
      - ./models/${MODEL_NAME:-llama3-8b-cpt-sea-lionv2.1-instruct}:/models/${MODEL_NAME:-llama3-8b-cpt-sea-lionv2.1-instruct}
    deploy:
      resources:
        reservations:
          devices:
            - capabilities: ["gpu"]
    command: [
      "--dtype", "half",
      "--max-model-len", "512",
      "--enforce-eager",
      "--served-model-name", "${MODEL_NAME:-llama3-8b-cpt-sea-lionv2.1-instruct}",
      "--model", "/models/${MODEL_NAME:-llama3-8b-cpt-sea-lionv2.1-instruct}"
    ]