Hcompany/Holo-3.1-35B-A3B-GGUF

硬體：NVIDIA DGX SparkGPU：NVIDIA GB10系統：Ubuntu Linux模型儲存位置：/mnt/ai-models推論框架：llama.cpp模型格式：GGUF量化版本：Q4_K_M

Hcompany/Holo-3.1-35B-A3B-NVFP4

Loading safetensors checkpoint shards: 100% Completed | 3/3Loading weights took 142.78 seconds

NotImplementedError:Could not run '_C::gptq_marlin_repack'with arguments from the 'CUDA' backend

/mnt/ai-models/huggingface/hub/

find /mnt/ai-models/huggingface \  -maxdepth 3 \  -type d \  -name 'models--Hcompany--Holo-3.1-35B-A3B-NVFP4' \  -print

du -sh \  /mnt/ai-models/huggingface/hub/models--Hcompany--Holo-3.1-35B-A3B-NVFP4

rm -rf \  /mnt/ai-models/huggingface/hub/models--Hcompany--Holo-3.1-35B-A3B-NVFP4

sudo apt update
sudo apt install -y \  git \  build-essential \  cmake \  ninja-build \  libcurl4-openssl-dev \  pkg-config

export CUDA_HOME=/usr/local/cudaexport PATH="${CUDA_HOME}/bin:${PATH}"
export LD_LIBRARY_PATH="${CUDA_HOME}/lib64:${LD_LIBRARY_PATH:-}"
nvcc --versionnvidia-smi

mkdir -p /mnt/ai-models/srccd /mnt/ai-models/srcgit 
clone https://github.com/ggml-org/llama.cpp.gitcd 
llama.cpp

cd /mnt/ai-models/src/llama.cpp
git fetch origingit switch master
git pull --ff-only origin master

cd /mnt/ai-models/src/llama.cpp
rm -rf buildcmake -B build \  -DGGML_CUDA=ON \  -DCMAKE_BUILD_TYPE=Releasecmake --build build \  --config Release \  -j 4 \  --target llama-server llama-cli

./build/bin/llama-server --version
./build/bin/llama-cli --version

mkdir -p \  /mnt/ai-models/llama-models/Holo-3.1-35B-A3B-GGUF

hf download \  Hcompany/Holo-3.1-35B-A3B-GGUF \  q4_k_m.gguf \  mmproj.f16.gguf \  chat_template.jinja \  --local-dir \  /mnt/ai-models/llama-models/Holo-3.1-35B-A3B-GGUF

ls -lh \  /mnt/ai-models/llama-models/Holo-3.1-35B-A3B-GGUF

q4_k_m.ggufmmproj.f16.ggufchat_template.jinja

cd /mnt/ai-models/src/llama.cpp
./build/bin/llama-server \  -m /mnt/ai-models/llama-models/Holo-3.1-35B-A3B-GGUF/q4_k_m.gguf \  --mmproj /mnt/ai-models/llama-models/Holo-3.1-35B-A3B-GGUF/mmproj.f16.gguf \  --jinja \  --host 0.0.0.0 \  --port 8080 \  -c 8192 \  -np 1 \  -ngl 999

--mmproj     指定視覺投影模型
--jinja      使用模型附帶的 Chat Template
-c 8192      Context Length
-np 1        同時處理 1 個請求
-ngl 999     儘可能將模型層放到 GPU

curl -s http://192.168.0.240:8080/v1/chat/completions \  -H "Content-Type: application/json" \  -d '{    "model": "Holo-3.1-35B-A3B-GGUF",    "messages": [      {        "role": "user",        "content": "請使用繁體中文介紹你的 UI 畫面分析能力。"      }    ],    "max_tokens": 256,    "temperature": 0.2  }' | python3 -m json.tool

82 tokens/s

curl -s http://127.0.0.1:8080/v1/chat/completions \  -H "Content-Type: application/json" \  -d '{    "model": "q4_k_m.gguf",    "messages": [      {        "role": "user",        "content": [          {            "type": "text",            "text": "Describe this image in one concise English sentence."          },          {            "type": "image_url",            "image_url": {              "url": "https://cdn.britannica.com/61/93061-050-99147DCE/Statue-of-Liberty-Island-New-York-Bay.jpg"            }          }        ]      }    ],    "thinking_budget_tokens": 64,    "max_tokens": 1024,    "temperature": 0.1  }' | python3 -m json.tool

Failed to parse input at pos ...

自由女神像、火炬、基座、城市天際線、高樓、水面、島嶼、樹木、旗�、船。

1. 降低 temperature，例如 0.12. 限制輸出為簡短句子3. 提高 max_tokens，避免輸出被截斷4. Client 端遇到 500 時自動 Retry 一次5. 優先更新至最新版 llama.cpp

cd /mnt/ai-models/src/llama.cpp
./build/bin/llama-server \  --models-dir /mnt/ai-models/llama-models \  --models-max 1 \  --models-autoload \  --jinja \  --host 0.0.0.0 \  --port 8080 \  -c 8192 \  -np 1 \  -ngl 999

Loaded 1 local model presets from /mnt/ai-models/llama-modelsAvailable models (1)    Holo-3.1-35B-A3B-GGUFstarting router server, no model will be loaded in this processrouter server is listening on http://0.0.0.0:8080

curl -s \  'http://127.0.0.1:8080/models?reload=1' | \  python3 -m json.tool

curl -s http://127.0.0.1:8080/v1/chat/completions \  -H "Content-Type: application/json" \  -d '{    "model": "Holo-3.1-35B-A3B-GGUF",    "messages": [      {        "role": "user",        "content": "請使用繁體中文簡短介紹你的功能。"      }    ],    "max_tokens": 512,    "temperature": 0.2  }' | python3 -m json.tool

--models-max 1

可以讓 Client 選擇多個模型但同一時間只保留一個模型在記憶體中

Holo 35B：Context 8192 或 16384用途：UI 截圖與視覺分析較小的文字 Tool Calling 模型：Context 65536用途：Hermes Agent 預設模型

version = 1

[*]
jinja = true
n-gpu-layers = 999
parallel = 1

[Holo-3.1-35B-A3B-GGUF]
model = /mnt/ai-models/llama-models/Holo-3.1-35B-A3B-GGUF/q4_k_m.gguf
mmproj = /mnt/ai-models/llama-models/Holo-3.1-35B-A3B-GGUF/mmproj.f16.gguf
ctx-size = 8192

[qwen-coder]
model = /mnt/ai-models/llama-models/qwen-coder-14b-q4_k_m.gguf
ctx-size = 65536

./build/bin/llama-server \  --models-preset /mnt/ai-models/llama-models/models.ini \  --models-max 1 \  --models-autoload \  --host 0.0.0.0 \  --port 8080

ls -lh /mnt/ai-models/src/llama.cpp/build/bin/llama-server

ls -lah /mnt/ai-models/llama-models

/mnt/ai-models/src/llama.cpp/build/bin/llama-server --version

sudo nano /etc/systemd/system/llama-router.service

[Unit]
Description=llama.cpp Router Server
Documentation=https://github.com/ggml-org/llama.cpp
After=network-online.target local-fs.target
Wants=network-online.target
RequiresMountsFor=/mnt/ai-models

[Service]
Type=simple
User=gwoyju
Group=gwoyju
WorkingDirectory=/mnt/ai-models/src/llama.cpp

Environment="LD_LIBRARY_PATH=/usr/local/cuda/lib64"
Environment="CUDA_HOME=/usr/local/cuda"

ExecStartPre=/usr/bin/test -x /mnt/ai-models/src/llama.cpp/build/bin/llama-server
ExecStartPre=/usr/bin/test -d /mnt/ai-models/llama-models

ExecStart=/mnt/ai-models/src/llama.cpp/build/bin/llama-server \
  --models-dir /mnt/ai-models/llama-models \
  --models-max 1 \
  --models-autoload \
  --jinja \
  --host 0.0.0.0 \
  --port 8080 \
  -c 8192 \
  -np 1 \
  -ngl 999

Restart=on-failure
RestartSec=10
TimeoutStopSec=30
KillSignal=SIGTERM
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target

Ctrl + O
Enter
Ctrl + X

/mnt/ai-models

RequiresMountsFor=/mnt/ai-models

sudo systemctl daemon-reload

sudo systemctl enable --now llama-router

systemctl status llama-router --no-pager

Active: active (running)

router server is listening on http://0.0.0.0:8080

journalctl -u llama-router -n 100 --no-pager

journalctl -u llama-router -f

Ctrl + C

curl -s http://127.0.0.1:8080/health

curl -s \  'http://127.0.0.1:8080/models?reload=1' | \  python3 -m json.tool

curl -s http://127.0.0.1:8080/v1/chat/completions \  -H "Content-Type: application/json" \  -d '{    "model": "Holo-3.1-35B-A3B-GGUF",    "messages": [      {        "role": "user",        "content": "請使用繁體中文簡短介紹你的功能。"      }    ],    "max_tokens": 256,    "temperature": 0.2  }' | python3 -m json.tool

sudo systemctl start llama-router

sudo systemctl stop llama-router

sudo systemctl restart llama-router

systemctl status llama-router --no-pager

sudo systemctl disable --now llama-router

sudo reboot

systemctl status llama-router --no-pager

ss -ltnp | grep :8080

curl -s http://127.0.0.1:8080/models | \  python3 -m json.tool

Holo 35B：8192Hermes 預設 Tool Calling 模型：65536

--models-dir-c 8192

--models-preset

/mnt/ai-models/llama-models/models.ini

ExecStart=/mnt/ai-models/src/llama.cpp/build/bin/llama-server \
  --models-preset /mnt/ai-models/llama-models/models.ini \
  --models-max 1 \
  --models-autoload \
  --host 0.0.0.0 \
  --port 8080

sudo systemctl daemon-reload
sudo systemctl restart llama-router

journalctl -u llama-router -f

sudo systemctl disable --now ollama

systemctl status ollama --no-pager

sudo systemctl enable --now ollama

--host 0.0.0.0

192.168.0.0/24

sudo ufw allow from 192.168.0.0/24 \  to any port 8080 proto tcp

sudo ufw status numbered

sudo systemctl daemon-reload
sudo systemctl enable --now llama-router
systemctl status llama-router --no-pagerjournalctl -u llama-router -n 50 --no-pager