|
@@ -182,37 +182,30 @@ args = [
|
|
|
# knobs). The card's --kv-cache-dtype fp8_e4m3 is the fallback spelling
|
|
# knobs). The card's --kv-cache-dtype fp8_e4m3 is the fallback spelling
|
|
|
# if the pool needs to shrink.
|
|
# if the pool needs to shrink.
|
|
|
"--attention-backend", "B12X",
|
|
"--attention-backend", "B12X",
|
|
|
- # MXFP4 MoE path — MUST be explicit: the r35/r37 tree's DEFAULT moe
|
|
|
|
|
- # backend is b12x (r35 changelog), which is the W4A4 path we use for the
|
|
|
|
|
- # DS4.1 MXFP4 experts, not the route the lab qualifies for MXFP4 GLM
|
|
|
|
|
- # experts. Their GLM-5.1 MXFP4 runbook (models/glm-5.1-mxfp4.md,
|
|
|
|
|
- # "Runtime Rules") is explicit: `--moe-backend flashinfer_cutlass_afp8`
|
|
|
|
|
- # selects the native FlashInfer Cutlass MXFP4xMXFP8 MoE path, and
|
|
|
|
|
- # B12X_MOE_FORCE_A16 must NOT be set for that checkpoint. Verified present
|
|
|
|
|
- # in OUR Sept tree (read-only image inspection 2026-09-13):
|
|
|
|
|
- # fused_moe/oracle/mxfp4.py:297 "flashinfer_cutlass_afp8" ->
|
|
|
|
|
- # Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8
|
|
|
|
|
- # quantization/mxfp4.py:79/134 non-gpt_oss routes to the native
|
|
|
|
|
- # Mxfp4MoEMethod (GptOssMxfp4Config abstains)
|
|
|
|
|
- # fused_moe/experts/flashinfer_cutlass_moe.py:270-280 the gpt-oss
|
|
|
|
|
- # alpha/beta/clamp SwiGLU defaults are gated
|
|
|
|
|
- # -> GLM gets standard SwiGLU
|
|
|
|
|
- # so the June GLM-5.1 MXFP4 vLLM changes are merged upstream already.
|
|
|
|
|
- # FALLBACK if this backend rejects the compressed-tensors checkpoint
|
|
|
|
|
- # (RedHatAI ships quant_method=compressed-tensors / format=
|
|
|
|
|
- # mxfp4-pack-quantized, i.e. CompressedTensorsW4A4Mxfp4 — a different
|
|
|
|
|
- # ingestion path from the lab's quant_method=mxfp4): try "b12x" (proven on
|
|
|
|
|
- # our DS4.1 MXFP4 experts) or drop the flag and read the engine's selected
|
|
|
|
|
- # MoE backend from the log. WHICHEVER backend runs, validate the numerics:
|
|
|
|
|
- # the missing-SwiGLU-clamp class of bug (r33/r34: runaway generation) and
|
|
|
|
|
- # the W4A4 token-loop corruption both show up as degenerate repeats.
|
|
|
|
|
- # (The lab runbook's literal `flashinfer_cutlass_afp8` was a CLI choice in
|
|
|
|
|
- # their June tree. The Sept tree's CLI coarsens it to the `flashinfer_cutlass`
|
|
|
|
|
- # family, which the oracle (fused_moe/oracle/mxfp4.py) maps to
|
|
|
|
|
- # [FLASHINFER_CUTLASS_MXFP4_BF16, FLASHINFER_CUTLASS_MXFP4_MXFP8] with the
|
|
|
|
|
- # `_afp8` activation key resolving the MXFP8 kernel. The exact string is
|
|
|
|
|
- # rejected by argparse on r35 — verified on the first boot attempt.)
|
|
|
|
|
- "--moe-backend", "flashinfer_cutlass",
|
|
|
|
|
|
|
+ # MXFP4 kernel routing (measured on the first boot, 2026-09-13 ~23:10 UTC):
|
|
|
|
|
+ # - MoE: the compressed-tensors W4A4-MXFP4 scheme picks MarlinExperts on
|
|
|
|
|
+ # its own (A16 dequant GEMM - correct on GB10). The --moe-backend flag
|
|
|
|
|
+ # does not reach that scheme (verified inert: it still logged "Using
|
|
|
|
|
+ # MarlinExperts for MXFP4 MoE" with the flag set), so we do not set it.
|
|
|
|
|
+ # - LINEAR: the default priority [FlashInfer, Marlin, Humming, B12x]
|
|
|
|
|
+ # starts with the FlashInfer kernel, whose is_supported() wrongly
|
|
|
|
|
+ # accepts capability 121 (the numeric has_device_capability(100) check
|
|
|
|
|
+ # passes on GB10) while the cute-dsl mm_fp4 GEMM it calls rejects it at
|
|
|
|
|
+ # runtime with:
|
|
|
|
|
+ # BackendSupportedError: mm_fp4 does not support backend 'cute-dsl'
|
|
|
|
|
+ # with capability 121
|
|
|
|
|
+ # (the first probe boot died there, kv_b_proj dequant in mla_attention.py).
|
|
|
|
|
+ # --linear-backend b12x forces the B12xMxFp4LinearKernel - "MXFP4 linear
|
|
|
|
|
+ # through the native B12X SM120 dense GEMM" - the right path on GB10,
|
|
|
|
|
+ # verified per component: is_device_capability_family(120)=True, b12x
|
|
|
|
|
+ # blockscaled.is_supported()=True, and the cute-dsl input quantize
|
|
|
|
|
+ # (flashinfer_mxfp4_quantize) works on capability 121; only the cute-dsl
|
|
|
|
|
+ # mm_fp4 GEMM is unsupported there. Boot log to look for:
|
|
|
|
|
+ # "Using B12xMxFp4LinearKernel for MXFP4 GEMM"
|
|
|
|
|
+ # (The lab's GLM-5.1 runbook describes the same kernel family for the
|
|
|
|
|
+ # quant_method=mxfp4 ingestion path - this checkpoint goes through the
|
|
|
|
|
+ # compressed-tensors path, which is why --moe-backend was the wrong lever.)
|
|
|
|
|
+ "--linear-backend", "b12x",
|
|
|
# pure I/O loader (kept across every GLM config)
|
|
# pure I/O loader (kept across every GLM config)
|
|
|
"--load-format", "instanttensor",
|
|
"--load-format", "instanttensor",
|
|
|
"--block-size", "256",
|
|
"--block-size", "256",
|
