|
@@ -0,0 +1,38 @@
|
|
|
|
|
+engine = "vllm"
|
|
|
|
|
+name = "Gemma 4 12B IT (MTP)"
|
|
|
|
|
+docker_image = "vllm/vllm-openai:gemma4-unified"
|
|
|
|
|
+model = "google/gemma-4-12B-it"
|
|
|
|
|
+# docker_image = "ghcr.io/timothystewart6/vllm-gb10:v0.23.0-gb10.1"
|
|
|
|
|
+# model = "RedHatAI/gemma-4-12B-it-NVFP4"
|
|
|
|
|
+tensor_parallel_size = 1
|
|
|
|
|
+pipeline_parallel_size = 1
|
|
|
|
|
+
|
|
|
|
|
+# Single-node recipe: Gemma 4 12B with BF16, FP8 KV cache,
|
|
|
|
|
+# MTP speculative decoding (5 tokens), 196K context.
|
|
|
|
|
+
|
|
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
|
|
+
|
|
|
|
|
+args = [
|
|
|
|
|
+ "--dtype", "bfloat16",
|
|
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
|
|
+ "--gpu-memory-utilization", "0.75",
|
|
|
|
|
+ "--max-model-len", "196608",
|
|
|
|
|
+ "--max-num-seqs", "2",
|
|
|
|
|
+ "--max-num-batched-tokens", "8192",
|
|
|
|
|
+ "--enable-prefix-caching",
|
|
|
|
|
+ "--enable-chunked-prefill",
|
|
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
|
|
+ "--tool-call-parser", "gemma4",
|
|
|
|
|
+ "--reasoning-parser", "gemma4",
|
|
|
|
|
+ "--generation-config", "vllm",
|
|
|
|
|
+ "--safetensors-load-strategy", "prefetch",
|
|
|
|
|
+ "--speculative-config", """
|
|
|
|
|
+'{"method":"mtp","model":"google/gemma-4-12B-it-assistant","num_speculative_tokens":5}'""",
|
|
|
|
|
+ "--limit-mm-per-prompt", """
|
|
|
|
|
+'{"image":4,"audio":1,"video":1}'""",
|
|
|
|
|
+ "--trust-remote-code",
|
|
|
|
|
+ "--served-model-name", "gemma4-12b",
|
|
|
|
|
+]
|
|
|
|
|
+
|
|
|
|
|
+[env]
|
|
|
|
|
+TRITON_CACHE_DIR = "/root/.cache/triton"
|
