|
@@ -0,0 +1,58 @@
|
|
|
|
|
+# Ling-3.1-flash fp4 (inclusionAI) — OUR OWN fp8→MXFP4 requant, 4x DGX Spark.
|
|
|
|
|
+#
|
|
|
|
|
+# Built by our converter (ling_fp4_full.py): routed experts (layers 4-72)
|
|
|
|
|
+# requantized fp8_block_128x128 → MXFP4 (e2m1 packed + e8m0 group-32 scales,
|
|
|
|
|
+# vLLM reference_mxfp4 "even" scale mode); dense fp8 + bf16 modules kept
|
|
|
|
|
+# byte-identical. Finalized 2026-10-17: 214025 tensors, 313.2 GB, 54 shards.
|
|
|
|
|
+# config.json: routed_experts_quant_method=mxfp4, scale_dtype=e8m0,
|
|
|
|
|
+# weight_dtype=e2m1, source=fp8_block_128x128, NO scale_fmt (dense fp8 scales
|
|
|
|
|
+# stay F32 — vLLM is_scale_e8m0 is driven by scale_fmt==ue8m0, we keep F32).
|
|
|
|
|
+#
|
|
|
|
|
+# SIZE CORRECTION: an earlier note guessed fp4 ≈ 110 GB → one Spark. WRONG.
|
|
|
|
|
+# The routed experts dominate (512 experts × 68 layers); fp4 only shrinks them
|
|
|
|
|
+# ~0.53×, so the checkpoint is 313 GB → TP4 (78 GB weights/rank), NOT one Spark.
|
|
|
|
|
+# Still a big win: 313 GB fp4 on 4 Sparks vs 534 GB fp8 on 8.
|
|
|
|
|
+#
|
|
|
|
|
+# Base: stock `vllm/vllm-openai:v0.31.0` — the Ling fp4 loader is built in
|
|
|
|
|
+# (bailing_moe_v3.py sets store_dtype=mxfp4 when routed_experts_quant_method==
|
|
|
|
|
+# mxfp4 → Mxfp4MoEMethod; _LING_MXFP4_WEIGHTS_MAPPER maps weight_scale_inv →
|
|
|
|
|
+# weight_scale). No custom image. HF_HUB_OFFLINE=1: this repo is NOT on the HF
|
|
|
|
|
+# hub (local-only requant), so force cache-only resolution.
|
|
|
|
|
+#
|
|
|
|
|
+# TP4 chosen (not TP6) to leave 2 Sparks for the GLM-5.3-Flash-FP4-Local 2x
|
|
|
|
|
+# lane on the same 6 free nodes. 78 GB weights/rank at GMU 0.78 leaves ~15 GB
|
|
|
|
|
+# KV (MLA KV is compact) — if the GB10 UMA boot gate (vLLM #48140) refuses at
|
|
|
|
|
+# 65536, drop --max-model-len to 32768.
|
|
|
|
|
+#
|
|
|
|
|
+# Quality A/B note: tool-eval-bench quality is TP-invariant (same weight math),
|
|
|
|
|
+# so TP4 fp4 vs the fp8 (TP8) isolates the quantization delta. A valid fp8
|
|
|
|
|
+# anchor was never benched (the 10-16 fp8 run scored 67/100 at 3.1% completion —
|
|
|
|
|
+# thinking + 120 s timeout excluded 94/97 scenarios). This fp4 run uses the
|
|
|
|
|
+# ling3 parser + harness --no-think for a valid absolute score.
|
|
|
|
|
+
|
|
|
|
|
+engine = "vllm"
|
|
|
|
|
+name = "Ling-3.1-flash fp4 (4x, our requant, stock v0.31.0)"
|
|
|
|
|
+docker_image = "vllm/vllm-openai:v0.31.0"
|
|
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
|
|
+model = "inclusionAI/Ling-3.1-flash-fp4"
|
|
|
|
|
+parent_model = "inclusionAI/Ling-3.1-flash"
|
|
|
|
|
+port = 30000
|
|
|
|
|
+required_nodes = 4
|
|
|
|
|
+tensor_parallel_size = 4
|
|
|
|
|
+pipeline_parallel_size = 1
|
|
|
|
|
+distributed_backend = "mp"
|
|
|
|
|
+args = [
|
|
|
|
|
+ "--served-model-name", "inclusionAI/Ling-3.1-flash-fp4",
|
|
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
|
|
+ "--tool-call-parser", "ling3",
|
|
|
|
|
+ "--trust-remote-code",
|
|
|
|
|
+ "--max-model-len", "65536",
|
|
|
|
|
+ "--gpu-memory-utilization", "0.78",
|
|
|
|
|
+ "--max-num-seqs", "32",
|
|
|
|
|
+ "--max-num-batched-tokens", "8192",
|
|
|
|
|
+]
|
|
|
|
|
+
|
|
|
|
|
+[env]
|
|
|
|
|
+VLLM_ENGINE_READY_TIMEOUT_S = "3600"
|
|
|
|
|
+# local-only repo (not on HF hub) — resolve from the mounted HF cache only
|
|
|
|
|
+HF_HUB_OFFLINE = "1"
|
