Преглед на файлове

fix(recipe): exclude 5 mis-packed layers from NF4 Lloyd decode set

Per the model card (2026-07-27), 5 experts across layers 5, 8, 9, 32, 46
were packed with E2M1/MXFP4 nibbles instead of Lloyd NF4 — the quantizer
selected its arm per-expert but decoding is per-layer. Routing those
layers through the Lloyd NF4 decode path introduces ~10.5% relative
weight error on the mis-packed expert.

Workaround: exclude layers 5, 8, 9, 32, 46 from VLLM_NF4_LAYERS so they
decode as E2M1, matching their actual bytes. 75 → 70 layers in the NF4
set. Layer 10 had the same defect but was already caught + fixed by the
upstream packing guard, so it stays in the NF4 set.

Ref: huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
     ("Known defect: 5 experts are packed on the wrong grid")
Alexander Korolev преди 2 месеца
родител
ревизия
2a72c806e4
променени са 1 файла, в които са добавени 11 реда и са изтрити 1 реда
  1. 11 1
      recipes/glm52-nf4-lloyd-experts-gptq/recipe.toml

+ 11 - 1
recipes/glm52-nf4-lloyd-experts-gptq/recipe.toml

@@ -101,9 +101,19 @@ repo = "aidendle94/GLM-5.2-MXFP4-Experts-GPTQ"
 repo = "aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ"
 glob = "L*.safetensors"
 
+# NF4 layer list — all 75 MoE layers (3–77) EXCEPT the 5 layers with a
+# known packing defect (per model card 2026-07-27). Layers 5, 8, 9, 32, 46
+# each contain 1 expert whose bytes were packed with E2M1/MXFP4 nibbles
+# (quantizer arm won per-expert, but decoding is per-layer). Routing those
+# layers through the Lloyd NF4 decode path introduces ~10.5% relative weight
+# error on the mis-packed expert. Excluding them forces E2M1 decode, matching
+# their actual bytes. Layer 10 had the same defect but was already caught +
+# fixed upstream by the packing guard, so it stays in the NF4 set.
+# Ref: https://huggingface.co/aidendle94/GLM-5.2-NF4-Lloyd-Experts-GPTQ
+# ("Known defect: 5 experts are packed on the wrong grid")
 [[composed_model.files]]
 name = "nf4_layers.json"
-content = '{"layers":[3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77]}'
+content = '{"layers":[3,4,6,7,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,33,34,35,36,37,38,39,40,41,42,43,44,45,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77]}'
 
 # Persistent JIT/torch-inductor cache (avoids ~20 min recompilation on cold boot).
 # `alias = "jit_cache"` resolves via [mount_aliases] in config.toml; if absent,