| 1234567891011121314151617181920212223242526272829303132333435363738 |
- engine = "vllm"
- name = "Gemma 4 12B IT (MTP)"
- docker_image = "vllm/vllm-openai:gemma4-unified"
- model = "google/gemma-4-12B-it"
- # docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
- # model = "RedHatAI/gemma-4-12B-it-NVFP4"
- tensor_parallel_size = 1
- pipeline_parallel_size = 1
- # Single-node recipe: Gemma 4 12B with BF16, FP8 KV cache,
- # MTP speculative decoding (5 tokens), 196K context.
- entrypoint = ["vllm", "serve"]
- args = [
- "--dtype", "bfloat16",
- "--kv-cache-dtype", "fp8",
- "--gpu-memory-utilization", "0.75",
- "--max-model-len", "196608",
- "--max-num-seqs", "2",
- "--max-num-batched-tokens", "8192",
- "--enable-prefix-caching",
- "--enable-chunked-prefill",
- "--enable-auto-tool-choice",
- "--tool-call-parser", "gemma4",
- "--reasoning-parser", "gemma4",
- "--generation-config", "vllm",
- "--safetensors-load-strategy", "prefetch",
- "--speculative-config", """
- '{"method":"mtp","model":"google/gemma-4-12B-it-assistant","num_speculative_tokens":5}'""",
- "--limit-mm-per-prompt", """
- '{"image":4,"audio":1,"video":1}'""",
- "--trust-remote-code",
- "--served-model-name", "gemma4-12b",
- ]
- [env]
- TRITON_CACHE_DIR = "/root/.cache/triton"
|
