launcher/recipes/qwen38-flash-next-int4/recipe.toml

# Qwen3.8-Flash-Next-125B-A5B INT4-AutoRound (azampatti) on a single DGX
# Spark — the "5 experts per token" cut with the healed shared expert.
#
# Source: forum 382733 (posts 167-171, 2026-09-25) + github.com/azampatti/
# Qwen3.8-Flash-Next-Int4-FAST (recipe repo, public). Numbers: 60-75 tok/s
# single Spark, ~645K-token KV pool at 20 GB, tool-eval 93/100 (his 69-scen
# v2.6-era harness; we re-bench on v2.7.0). BENCH NOTE: his quant defaults
# to MEDIUM thinking (the stock model is xhigh) → the chat template matters
# for A/B fairness (three switchable templates ship in the model repo; the
# served view mounts medium_chat_template.jinja — the template the
# speculative head was trained on).
#
# Image: eugr/spark-vllm-b12x (Docker Hub; the b12x stack his 2x recipe uses,
# ~75 tok/s vs 70 on the classic image). Prebuilt — no docker_image_dir.
#
# Model: azampatti/Qwen3.8-Flash-Next-125B-A5B-INT4-AutoRound (130.8 GB)
# staged at /home/kilork/hf-local on the launch node (dgx5); serve with
# --hf-cache /home/kilork/hf-local.
#
# The mod (mods/flashnext-int4-b12x, vendored from his repo) checks the model
# in the container's HF cache, builds /workspace/flashnext/{model,draft-k10}
# (symlink views + small JSON, zero copy) and patches the container's vLLM
# (fp8 hybrid, int4 lm_head, loader index filter, draft x2, GDN fixes).
# It must run per node (single node here) before the engine.

engine = "vllm"
name = "Qwen3.8-Flash-Next-Int4-AutoRound (b12x, 1x Spark)"
docker_image = "eugr/spark-vllm-b12x:latest"
# Empty recipe model: the launcher emits no positional model, and args[0] is
# the mod-built view (vllm serves the view, not the raw repo — the view carries
# the PLE-table index + the b12x arch rename that the raw snapshot lacks).
model = ""
parent_model = "Qwen/Qwen3.8-Flash-Next"
port = 8000
required_nodes = 1

entrypoint = ["vllm", "serve"]

# vllm serves the mod-built view (not the raw repo): the launcher passes the
# recipe model to the mod via the HF cache; the mod builds /workspace/...
# and run.sh validates it. The command targets the view, per his yaml.
args = [
    "/workspace/flashnext/model",
    "--served-model-name", "azampatti/Qwen3.8-Flash-Next-125B-A5B-INT4-AutoRound",
    "--trust-remote-code",
    "--enable-expert-parallel",
    "--mamba-cache-mode", "align",
    "--linear-backend", "b12x",
    "--enable-prefix-caching",
    "--enable-chunked-prefill",
    "--dtype", "bfloat16",
    "--kv-cache-dtype", "auto",
    "--block-size", "16",
    "--load-format", "fastsafetensors",
    "--max-model-len", "262144",
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "8192",
    "--gpu-memory-utilization", "0.01",
    "--kv-cache-memory-bytes", "20g",
    "--gdn-decode-kernel", "b12x",
    "--no-enable-flashinfer-autotune",
    "--mm-encoder-tp-mode", "data",
    "--enable-auto-tool-choice",
    "--tool-call-parser", "qwen3_coder",
    "--reasoning-parser", "qwen3",
    "--chat-template", "/workspace/flashnext/model/medium_chat_template.jinja",
    "--speculative-config",
    '{"method":"mtp","num_speculative_tokens":4,"model":"/workspace/flashnext/draft-k10","rejection_sample_method":"block","draft_sample_method":"probabilistic"}',
]

[mods]
commands = ["bash /mods/flashnext-int4-b12x/run.sh"]

[env]
HF_HUB_OFFLINE = "1"
# b12x stack (his solo yaml, verbatim)
CUTE_DSL_ARCH = "sm_121a"
VLLM_WORKER_MULTIPROC_METHOD = "spawn"
VLLM_SSM_CONV_STATE_LAYOUT = "DS"
VLLM_USE_AOT_COMPILE = "1"
VLLM_USE_MEGA_AOT_ARTIFACT = "1"
VLLM_USE_V2_MODEL_RUNNER = "1"
B12X_POLICY_MODE = "auto"
VLLM_PLE_TABLE_MEMORY = "disk"
# this checkpoint (the mod patches them into the container's vLLM)
VLLM_FP8_HYBRID = "1"
VLLM_MARLIN_USE_ATOMIC_ADD = "1"
VLLM_USE_DEEP_GEMM = "0"
VLLM_USE_FLASHINFER_SAMPLER = "1"
VLLM_MTP_DRAFT_SCALE = "2"