launcher/recipes/solar-open2-nvfp4/recipe.toml

# Solar Open2 250B — Nota NVFP4 (compressed-tensors, W4A4)
# 2× DGX Spark (GB10, aarch64/sm_121a), TP=2 over RoCE.
#
# Model: nota-ai/Solar-Open2-250B-Nota-NVFP4
#   → https://huggingface.co/nota-ai/Solar-Open2-250B-Nota-NVFP4
#   145B params (MoE), 153.3 GB NVFP4 weights (vs 500.6 GB BF16).
#   NVFP4 is auto-detected from config (compressed-tensors) — no --quantization.
#
# Image: solar-open2-nvfp4:probe (built locally from the UpstageAI vLLM fork
#   v0.22.0-solar-open2 — carries the solar_open2 reasoning/tool parsers and
#   the SolarOpen2TemplateLogitsProcessor). cu130 / aarch64 / sm_121a, same
#   pinned base stack as glm52-nf3-hybrid (no b12x suite — stock vLLM serve).
#
# Memory: GB10 ≈ 121 GB usable (128 GB advertised − 4 GB OS). TP=2 →
#   ~77 GB weights/node, ~40 GB/node for KV cache + activations.
#
# PREREQUISITES:
#   - Weights cached on every node:
#       huggingface-cli download nota-ai/Solar-Open2-250B-Nota-NVFP4
#     (153 GB, NVFP4 compressed-tensors shards)
#   - Image built (sm121 serve builds it automatically on the build node):
#       cd docker-images/solar-open2-nvfp4 && bash build.sh

engine = "vllm"
name = "Solar Open2 250B Nota NVFP4 (vLLM)"
docker_image = "solar-open2-nvfp4:probe"
model = "nota-ai/Solar-Open2-250B-Nota-NVFP4"
parent_model = "upstage/Solar-Open2-250B"
required_nodes = 2
tensor_parallel_size = 2
pipeline_parallel_size = 1
distributed_backend = "mp"
shm_size = "32gb"
docker_image_dir = "solar-open2-nvfp4"

entrypoint = ["vllm", "serve"]

args = [
    # KV cache & memory
    "--kv-cache-dtype", "fp8",
    "--load-format", "auto",
    # Context length (128K; model supports up to 131584)
    "--max-model-len", "131072",
    # Batch & memory
    "--max-num-seqs", "8",
    "--max-num-batched-tokens", "4096",
    "--max-cudagraph-capture-size", "16",
    "--gpu-memory-utilization", "0.88",
    # Scheduling
    "--async-scheduling",
    "--enable-chunked-prefill",
    "--enable-prefix-caching",
    # Solar Open2 parsers + logits processor (ships in the UpstageAI fork)
    "--reasoning-parser", "solar_open2",
    "--tool-call-parser", "solar_open2",
    "--enable-auto-tool-choice",
    "--logits-processors", "vllm.v1.sample.logits_processor.solar_open2:SolarOpen2TemplateLogitsProcessor",
    "--default-chat-template-kwargs", """
'{"think_render_option":"preserved"}'""",
    # General
    "--trust-remote-code",
    "--served-model-name", "solar",
]

[env]
# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
TORCH_CUDA_ARCH_LIST = "12.1a"
CUTE_DSL_ARCH = "sm_121a"
FLASHINFER_CUDA_ARCH_LIST = "12.1a"

# ── Offline mode (weights must be cached on each node) ────────────────────
HF_HOME = "/root/.cache/huggingface"
HF_HUB_OFFLINE = "1"
TRANSFORMERS_OFFLINE = "1"

# ── Timeout safety net (cold boot: cudagraph + JIT) ───────────────────────
VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "3600"

# ── NCCL / RoCE (cluster config ib_hca/socket_ifname used for most) ──────
NCCL_CROSS_NIC = "1"
NCCL_CUMEM_ENABLE = "0"
NCCL_IGNORE_CPU_AFFINITY = "1"
NCCL_DEBUG = "WARN"

# ── Compute ────────────────────────────────────────────────────────────────
OMP_NUM_THREADS = "4"