# Qwen3.8-Flash-Next-FP8 (SGLang day-0) — 2× DGX Spark / GB10 (sm_121), TP=2.
#
# Qwen4-architecture preview: 125B-A6B MoE (512 experts, 10 routed + 1 shared),
# Gated DeltaNet + Qwen Sparse Attention, 51B n-gram "engram" table, 4B
# in-checkpoint MTP. FP8 fine-grained (block 128). Text + vision. Native 262K
# ctx (YaRN to 1M). Model: https://huggingface.co/Qwen/Qwen3.8-Flash-Next-FP8
#
# Engine: SGLang day-0 — the ONLY engine with qwen4_exp today. vLLM support
# is an open PR (#53896), merged into no GB10 fork image (aidendle94 / eugr /
# local-inference-lab are all pre-release pins).
#
# Image: lmsysorg/sglang:qwen38flashnext — multi-arch incl. arm64 (digest
# sha256:14ed58251…; re-verify at serve time). Day-0 = boots, not tuned.
#
# Memory: 172.8 GiB weights → 86.4 GiB/node. SGLang claims the 51B engram
# table offloads to host RAM (async prefetch) — if active, the GPU budget
# shrinks and a larger KV pool / longer ctx is possible.
#
# PREREQUISITES:
#   Weights cached on every node (dgx1/dgx2 ~/.cache/huggingface → NAS):
#       huggingface-cli download Qwen/Qwen3.8-Flash-Next-FP8
#
# REFERENCE:
#   https://forums.developer.nvidia.com/t/qwen3-8-flash-next/381228

engine = "sglang"
name = "Qwen3.8-Flash-Next FP8 (SGLang day-0)"
docker_image = "lmsysorg/sglang:qwen38flashnext"
model = "Qwen/Qwen3.8-Flash-Next-FP8"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"

entrypoint = ["python3", "-m", "sglang.launch_server"]

args = [
    # Memory — weights ~86.4 GiB of 119.6 GiB/node; the rest goes to KV +
    # activations. Undervalue to be safe on day-0; raise after measuring.
    "--mem-fraction-static", "0.80",
    "--max-running-requests", "10",
    # GDN (DeltaNet) hybrid state pool
    "--max-mamba-cache-size", "30",
    # Attention / KV
    "--attention-backend", "flashinfer",
    "--kv-cache-dtype", "fp8_e4m3",
    "--mamba-ssm-dtype", "bfloat16",
    "--mamba-full-memory-ratio", "4.21",
    "--mamba-radix-cache-strategy", "extra_buffer_lazy",
    # Chunked prefill
    "--chunked-prefill-size", "8192",
    # Compile + decode graphs
    "--enable-torch-compile", "--torch-compile-max-bs", "4",
    "--cuda-graph-max-bs-decode", "4",
    "--num-continuous-decode-steps", "2",
    # Tokenizer / reasoning / tool calling (Qwen3 family)
    "--reasoning-parser", "qwen3",
    "--tool-call-parser", "qwen3_coder",
    # Observability
    "--enable-metrics",
    "--enable-cache-report",
]

[env]
# Offline — weights must be cached on each node
HF_HUB_OFFLINE = "1"

# Architecture
TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
OMP_NUM_THREADS = "4"