Merge pull request #14 from janhq/feat/model-converter-pipeline

feat: support for model converter CI

.github/workflows/convert-model-all-quant.yml

@@ -0,0 +1,153 @@
+name: Convert model to gguf with specified quant
+
+on:
+  workflow_dispatch:
+    inputs:
+      source_model_id:
+        description: "Source HuggingFace model ID to pull. For ex: meta-llama/Meta-Llama-3.1-8B-Instruct"
+        required: true
+      source_model_size:
+        description: "The model size. For ex: 8b"
+        required: true
+        type: string
+      target_model_id:
+        description: "Target HuggingFace model ID to push. For ex: llama3.1"
+        required: true
+        type: string
+      quantization_level:
+        description: "Quantization level (e.g., 'q4-km') or 'all' for all levels"
+        required: true
+        type: string
+        default: 'all'
+
+env:
+  USER_NAME: cortexso
+  SOURCE_MODEL_ID: ${{ inputs.source_model_id }}
+  SOURCE_MODEL_SIZE: ${{ inputs.source_model_size }}
+  TARGET_MODEL_ID: ${{ inputs.target_model_id }}
+  QUANT_LEVEL: ${{ inputs.quantization_level }}
+
+jobs:
+  converter:
+    runs-on: ubuntu-20-04-gguf
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4 # v4.1.7
+        with:
+          submodules: recursive
+
+      - name: Set up Python
+        uses: actions/setup-python@v5 # v5.1.1
+        with:
+          python-version: '3.12'
+          # architecture: 'x64'
+
+      - name: Cache Python packages
+        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.local/share/pip
+            .venv
+          key: ${{ runner.os }}-pip-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          pip3 install -r llama.cpp/requirements.txt
+          pip3 install hf-transfer
+          git lfs install
+
+      - name: Extract MODEL_NAME
+        run: |
+          SOURCE_MODEL_ID="${{ env.SOURCE_MODEL_ID }}"
+          MODEL_NAME="$(echo $SOURCE_MODEL_ID | rev | cut -d/ -f1 | rev)"
+          echo $MODEL_NAME
+          MODEL_NAME="$(echo $MODEL_NAME | tr '[:upper:]' '[:lower:]')"
+          echo $MODEL_NAME
+          echo "MODEL_NAME=$MODEL_NAME" >> $GITHUB_ENV
+
+      - name: Print environment variables
+        run: |
+          echo "SOURCE_MODEL_ID: ${{ env.SOURCE_MODEL_ID }}"
+          echo "MODEL_NAME: ${{ env.MODEL_NAME }}"
+      - name: Check file existence
+        id: check_files
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "/mnt/models/${{ env.MODEL_NAME }}/hf"
+
+
+      - name: Prepare folders
+        if: steps.check_files.outputs.files_exists == 'false'
+        run: |
+          mkdir -p /mnt/models/${{ env.MODEL_NAME }}/hf
+          
+      
+      - name: Download Hugging Face model
+        id: download_hf
+        if: steps.check_files.outputs.files_exists == 'false'
+        run: |
+          huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_READ }} --add-to-git-credential
+          huggingface-cli download --repo-type model --local-dir /mnt/models/${{ env.MODEL_NAME }}/hf ${{ env.SOURCE_MODEL_ID }}
+          huggingface-cli logout
+
+      - name: Build lib for quantize
+        run: |
+          cd llama.cpp && make
+          cd ../../
+
+      - name: Convert to GGUF
+        run: |
+          mkdir -p /mnt/models/${{ env.MODEL_NAME }}/gguf
+          huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_READ }} --add-to-git-credential
+          python3 llama.cpp/convert_hf_to_gguf.py "/mnt/models/${{ env.MODEL_NAME }}/hf" --outfile "/mnt/models/${{ env.MODEL_NAME }}/gguf/model-origin.gguf"
+          huggingface-cli logout
+      
+      - name: Quantize the model 
+        run: |
+          declare -A quant_map=(
+            ["q2-k"]="Q2_K"
+            ["q3-ks"]="Q3_K_S"
+            ["q3-km"]="Q3_K_M"
+            ["q3-kl"]="Q3_K_L"
+            ["q4-ks"]="Q4_K_S"
+            ["q4-km"]="Q4_K_M"
+            ["q5-ks"]="Q5_K_S"
+            ["q5-km"]="Q5_K_M"
+            ["q6-k"]="Q6_K"
+            ["q8-0"]="Q8_0"
+          )
+
+          if [ "${{ env.QUANT_LEVEL }}" = "all" ]; then
+            quant_levels=("q2-k" "q3-ks" "q3-km" "q3-kl" "q4-ks" "q4-km" "q5-ks" "q5-km" "q6-k" "q8-0")
+          else
+            quant_levels=("${{ env.QUANT_LEVEL }}")
+          fi
+
+          for quant in "${quant_levels[@]}"; do
+            mkdir -p /mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/
+            [ ! -f /mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/model.gguf ] && ./llama.cpp/llama-quantize /mnt/models/${{ env.MODEL_NAME }}/gguf/model-origin.gguf /mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/model.gguf ${quant_map[${quant}]}
+          done
+
+          rm -rf /mnt/models/${{ env.MODEL_NAME }}/gguf/model-origin.gguf
+
+      - name: Upload to Hugging Face
+        run: |
+          huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential
+          
+          if [ "${{ env.QUANT_LEVEL }}" = "all" ]; then
+            quant_levels=("q2-k" "q3-ks" "q3-km" "q3-kl" "q4-ks" "q4-km" "q5-ks" "q5-km" "q6-k" "q8-0")
+          else
+            quant_levels=("${{ env.QUANT_LEVEL }}")
+          fi
+
+          for quant in "${quant_levels[@]}"; do
+            huggingface-cli upload "${{ env.USER_NAME }}/${{ env.TARGET_MODEL_ID }}" "/mnt/models/${{ env.MODEL_NAME }}/gguf/${quant}/" . --revision "${{ env.SOURCE_MODEL_SIZE }}-gguf-${quant}"
+          done
+
+          rm -rf /mnt/models/${{ env.MODEL_NAME }}/gguf/*
+          huggingface-cli logout
+          rm -rf llama.cpp/build/
+

.github/workflows/update-model-yml.yml

@@ -0,0 +1,101 @@
+name: Update model.yml with specified quant
+
+on:
+  workflow_dispatch:
+    inputs:
+      key_value_pairs:
+        description: "the 'key=value' pairs that you want to update, separated by space"
+        required: true
+        type: string
+      target_model_id:
+        description: "Target HuggingFace model ID to update. For ex: llama3.1"
+        required: true
+        type: string
+      source_model_size:
+        description: "The model size. For ex: 8b"
+        required: true
+        type: string
+      quantization_level:
+        description: "Quantization level (e.g., 'q4-km') or 'all' for all levels"
+        required: true
+        type: string
+        default: 'all'
+
+env:
+  USER_NAME: cortexso
+  KEY_VALUE_PAIRS: ${{ inputs.key_value_pairs }}
+  SOURCE_MODEL_SIZE: ${{ inputs.source_model_size }}
+  TARGET_MODEL_ID: ${{ inputs.target_model_id }}
+  QUANT_LEVEL: ${{ inputs.quantization_level }}
+
+jobs:
+  model-yaml-updater:
+    runs-on: ubuntu-20-04-gguf
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Cache Python packages
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.local/share/pip
+            .venv
+          key: ${{ runner.os }}-pip-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+
+      - name: Install dependencies
+        run: |
+          pip3 install -I hf-transfer huggingface-hub ruamel.yaml
+          git lfs install
+
+      - name: Prepare folders
+        run: |
+          rm -rf /mnt/models/${{ env.MODEL_NAME }}/yaml/
+          mkdir -p /mnt/models/${{ env.MODEL_NAME }}/yaml
+
+      - name: Quantize and Upload
+        run: |
+          if [ "${{ env.QUANT_LEVEL }}" = "all" ]; then
+            quant_levels=("q2-k" "q3-ks" "q3-km" "q3-kl" "q4-ks" "q4-km" "q5-ks" "q5-km" "q6-k" "q8-0")
+          else
+            quant_levels=("${{ env.QUANT_LEVEL }}")
+          fi
+
+          for quant in "${quant_levels[@]}"; do
+            mkdir -p "/mnt/models/${{ env.MODEL_NAME }}/yaml/${quant}/"
+            python3 scripts/update_model_yml.py \
+              --repo_id "${{ env.USER_NAME }}/${{ env.TARGET_MODEL_ID }}" \
+              --filename model.yml \
+              --branch "${{ env.SOURCE_MODEL_SIZE }}-gguf-${quant}" \
+              --save_path "/mnt/models/${{ env.MODEL_NAME }}/yaml/${quant}/" \
+              --key_value_pairs name=${{ env.TARGET_MODEL_ID }}:${{ env.SOURCE_MODEL_SIZE }}-gguf-${quant} ${{ env.KEY_VALUE_PAIRS }}
+          done
+
+      - name: Upload to Hugging Face
+        run: |
+          huggingface-cli login --token ${{ secrets.HUGGINGFACE_TOKEN_WRITE }} --add-to-git-credential
+          
+          if [ "${{ env.QUANT_LEVEL }}" = "all" ]; then
+            quant_levels=("q2-k" "q3-ks" "q3-km" "q3-kl" "q4-ks" "q4-km" "q5-ks" "q5-km" "q6-k" "q8-0")
+          else
+            quant_levels=("${{ env.QUANT_LEVEL }}")
+          fi
+
+          for quant in "${quant_levels[@]}"; do
+            huggingface-cli upload "${{ env.USER_NAME }}/${{ env.TARGET_MODEL_ID }}" \
+              "/mnt/models/${{ env.MODEL_NAME }}/yaml/${quant}/" . \
+              --revision "${{ env.SOURCE_MODEL_SIZE }}-gguf-${quant}"
+          done
+
+          rm -rf /mnt/models/${{ env.MODEL_NAME }}/yaml/*
+          huggingface-cli logout

scripts/update_model_yml.py

@@ -0,0 +1,108 @@
+import argparse
+from huggingface_hub import hf_hub_download
+import os
+from ruamel.yaml import YAML
+import ast
+
+def download_yaml_from_huggingface(repo_id, filename, save_path, branch=None):
+    try:
+        # Download the file from the Hugging Face Hub
+        try:
+            downloaded_path = hf_hub_download(repo_id=repo_id, filename=filename, revision=branch)
+        except:
+            downloaded_path = hf_hub_download(repo_id=repo_id, filename=filename)
+
+        # Ensure the directory for save_path exists
+        os.makedirs(save_path, exist_ok=True)
+        full_save_path = os.path.join(save_path, "model.yml")
+        # Copy the downloaded file to the desired save location
+        with open(downloaded_path, 'rb') as src_file, open(full_save_path, 'wb') as dst_file:
+            dst_file.write(src_file.read())
+
+        print(f"File downloaded successfully and saved to {full_save_path}")
+        return full_save_path
+    except Exception as e:
+        print(f"An error occurred during download: {str(e)}")
+        return None
+
+def modify_nested_dict(data, keys, new_value):
+    current = data
+    for key in keys[:-1]:
+        if key not in current:
+            current[key] = {}
+        current = current[key]
+    current[keys[-1]] = new_value
+
+def modify_yaml(file_path, key_value_pairs):
+    yaml = YAML()
+    yaml.preserve_quotes = True
+    yaml.indent(mapping=2, sequence=4, offset=2)
+
+    try:
+        with open(file_path, 'r') as file:
+            data = yaml.load(file)
+
+        # Modify the specified fields
+        for field, new_value in key_value_pairs:
+            keys = field.split('.')
+            modify_nested_dict(data, keys, new_value)
+            print(f"Modified field '{field}' to '{new_value}'")
+
+        # Write the modified data back to the file
+        with open(file_path, 'w') as file:
+            yaml.dump(data, file)
+
+        print(f"Successfully modified all specified fields in {file_path}")
+    except Exception as e:
+        print(f"An error occurred while modifying the YAML file: {str(e)}")
+
+def parse_value(value):
+    try:
+        # Try to evaluate the string as a Python literal
+        parsed_value = ast.literal_eval(value)
+
+        # If it's a list, parse each element
+        if isinstance(parsed_value, list):
+            return [parse_value(item) for item in parsed_value]
+
+        return parsed_value
+    except (ValueError, SyntaxError):
+        # If it's not a valid Python literal, return it as a string
+        return value
+
+def parse_key_value_pair(pair):
+    try:
+        field, value = pair.split('=', 1)  # Split on first '=' only
+        parsed_value = parse_value(value)
+        return field, parsed_value
+    except ValueError:
+        raise argparse.ArgumentTypeError(f"Invalid field-value pair: {pair}. Use format 'field=value'")
+
+def main():
+    parser = argparse.ArgumentParser(description="Download a YAML file from Hugging Face, modify multiple fields, and save it")
+    parser.add_argument("--repo_id", required=True, help="The ID of the Hugging Face repository")
+    parser.add_argument("--filename", required=True, help="The name of the YAML file to download")
+    parser.add_argument("--branch", default=None, help="The branch to download from (default is main)")
+    parser.add_argument("--save_path", required=True, help="The local path where the file should be saved")
+    parser.add_argument("--key_value_pairs", required=True, nargs='+', type=parse_key_value_pair,
+                        help="Field-value pairs to modify. Format: field1=value1 field2=value2 ...")
+
+    args = parser.parse_args()
+
+    # Download the file
+    downloaded_file = download_yaml_from_huggingface(
+        repo_id=args.repo_id,
+        filename=args.filename,
+        save_path=args.save_path,
+        branch=args.branch
+    )
+
+    if downloaded_file:
+        # Modify the YAML file
+        modify_yaml(
+            file_path=downloaded_file,
+            key_value_pairs=args.key_value_pairs
+        )
+
+if __name__ == "__main__":
+    main()