|
|
@@ -0,0 +1,86 @@
|
|
|
+# Solar Open2 250B — Nota NVFP4 (compressed-tensors, W4A4)
|
|
|
+# 2× DGX Spark (GB10, aarch64/sm_121a), TP=2 over RoCE.
|
|
|
+#
|
|
|
+# Model: nota-ai/Solar-Open2-250B-Nota-NVFP4
|
|
|
+# → https://huggingface.co/nota-ai/Solar-Open2-250B-Nota-NVFP4
|
|
|
+# 145B params (MoE), 153.3 GB NVFP4 weights (vs 500.6 GB BF16).
|
|
|
+# NVFP4 is auto-detected from config (compressed-tensors) — no --quantization.
|
|
|
+#
|
|
|
+# Image: solar-open2-nvfp4:probe (built locally from the UpstageAI vLLM fork
|
|
|
+# v0.22.0-solar-open2 — carries the solar_open2 reasoning/tool parsers and
|
|
|
+# the SolarOpen2TemplateLogitsProcessor). cu130 / aarch64 / sm_121a, same
|
|
|
+# pinned base stack as glm52-nf3-hybrid (no b12x suite — stock vLLM serve).
|
|
|
+#
|
|
|
+# Memory: GB10 ≈ 121 GB usable (128 GB advertised − 4 GB OS). TP=2 →
|
|
|
+# ~77 GB weights/node, ~40 GB/node for KV cache + activations.
|
|
|
+#
|
|
|
+# PREREQUISITES:
|
|
|
+# - Weights cached on every node:
|
|
|
+# huggingface-cli download nota-ai/Solar-Open2-250B-Nota-NVFP4
|
|
|
+# (153 GB, NVFP4 compressed-tensors shards)
|
|
|
+# - Image built (sm121 serve builds it automatically on the build node):
|
|
|
+# cd docker-images/solar-open2-nvfp4 && bash build.sh
|
|
|
+
|
|
|
+engine = "vllm"
|
|
|
+name = "Solar Open2 250B Nota NVFP4 (vLLM)"
|
|
|
+docker_image = "solar-open2-nvfp4:probe"
|
|
|
+model = "nota-ai/Solar-Open2-250B-Nota-NVFP4"
|
|
|
+parent_model = "upstage/Solar-Open2-250B"
|
|
|
+required_nodes = 2
|
|
|
+tensor_parallel_size = 2
|
|
|
+pipeline_parallel_size = 1
|
|
|
+distributed_backend = "mp"
|
|
|
+shm_size = "32gb"
|
|
|
+docker_image_dir = "solar-open2-nvfp4"
|
|
|
+
|
|
|
+entrypoint = ["vllm", "serve"]
|
|
|
+
|
|
|
+args = [
|
|
|
+ # KV cache & memory
|
|
|
+ "--kv-cache-dtype", "fp8",
|
|
|
+ "--load-format", "auto",
|
|
|
+ # Context length (128K; model supports up to 131584)
|
|
|
+ "--max-model-len", "131072",
|
|
|
+ # Batch & memory
|
|
|
+ "--max-num-seqs", "8",
|
|
|
+ "--max-num-batched-tokens", "4096",
|
|
|
+ "--max-cudagraph-capture-size", "16",
|
|
|
+ "--gpu-memory-utilization", "0.88",
|
|
|
+ # Scheduling
|
|
|
+ "--async-scheduling",
|
|
|
+ "--enable-chunked-prefill",
|
|
|
+ "--enable-prefix-caching",
|
|
|
+ # Solar Open2 parsers + logits processor (ships in the UpstageAI fork)
|
|
|
+ "--reasoning-parser", "solar_open2",
|
|
|
+ "--tool-call-parser", "solar_open2",
|
|
|
+ "--enable-auto-tool-choice",
|
|
|
+ "--logits-processors", "vllm.v1.sample.logits_processor.solar_open2:SolarOpen2TemplateLogitsProcessor",
|
|
|
+ "--default-chat-template-kwargs", """
|
|
|
+'{"think_render_option":"preserved"}'""",
|
|
|
+ # General
|
|
|
+ "--trust-remote-code",
|
|
|
+ "--served-model-name", "solar",
|
|
|
+]
|
|
|
+
|
|
|
+[env]
|
|
|
+# ── Architecture (aarch64/sm_121a DGX Spark) ──────────────────────────────
|
|
|
+TORCH_CUDA_ARCH_LIST = "12.1a"
|
|
|
+CUTE_DSL_ARCH = "sm_121a"
|
|
|
+FLASHINFER_CUDA_ARCH_LIST = "12.1a"
|
|
|
+
|
|
|
+# ── Offline mode (weights must be cached on each node) ────────────────────
|
|
|
+HF_HOME = "/root/.cache/huggingface"
|
|
|
+HF_HUB_OFFLINE = "1"
|
|
|
+TRANSFORMERS_OFFLINE = "1"
|
|
|
+
|
|
|
+# ── Timeout safety net (cold boot: cudagraph + JIT) ───────────────────────
|
|
|
+VLLM_EXECUTE_MODEL_TIMEOUT_SECONDS = "3600"
|
|
|
+
|
|
|
+# ── NCCL / RoCE (cluster config ib_hca/socket_ifname used for most) ──────
|
|
|
+NCCL_CROSS_NIC = "1"
|
|
|
+NCCL_CUMEM_ENABLE = "0"
|
|
|
+NCCL_IGNORE_CPU_AFFINITY = "1"
|
|
|
+NCCL_DEBUG = "WARN"
|
|
|
+
|
|
|
+# ── Compute ────────────────────────────────────────────────────────────────
|
|
|
+OMP_NUM_THREADS = "4"
|
