2026-09-26_015216_aiden16-sglang-4x.recipe.toml 5.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139
  1. # DeepSeek-V4.1-Flash on 4x DGX Spark with Aiden's SGLang production-1.6
  2. # (aidenle.com/recipes/deepseek-v4-1-flash-sglang-4x-dgx-spark).
  3. #
  4. # Image: aidendle94/sparkrun-sglang-dsv41-gb10:production-1.6 (Docker Hub,
  5. # arm64, 33.3 GB) = lmsysorg/sglang:dev-dsv41 (SGLang team's own DSV4.1
  6. # image, unmodified) + Aiden's MIT overlay (import hooks: Engram row store
  7. # from node-local/shard files inside the CUDA graph, b12x SM12x FP8 dense
  8. # kernels, SM120 prefill paging, allocator flush, one-shot RDMA collectives,
  9. # request guard). Lineage: replaced the vLLM stack as Aiden's production
  10. # 2026-09-17 (prefill +30-45%, KV pool +23%); 1.2: 16 concurrent; 1.5: rank
  11. # crash fix + 2 GB10 speedups; 1.6 current. Measured by Aiden on 4x: decode
  12. # 107/87/36 tok/s (count/code/prose C1), prefill 3.3K tok/s @32K, KV pool
  13. # ~4.2M, 524K context, ~10 min to healthy.
  14. #
  15. # Model: deepseek-ai/DeepSeek-V4.1-Flash (510 GB, fp8 native) on the NAS —
  16. # serve with --hf-cache /mnt/nas/ai/huggingface. The checkpoint snapshot
  17. # dba1be0a... is also the Engram row source (SPARK_ENGRAM_DIR): all four
  18. # ranks read their rows directly from the NAS shards (Aiden's fallback path;
  19. # his measured config uses 47.2 GiB node-local sparse copies, which needs the
  20. # private engram_local.py — the NFS read costs ~3-5 ms/step extra, acceptable
  21. # for a validation run).
  22. #
  23. # Per-rank Engram row ranges (the table has 384,006,168 rows per layer,
  24. # layers 1 and 14; four contiguous ranges partition each table):
  25. # rank 0: 0..96001542 rank 1: 96001542..192003084
  26. # rank 2: 192003084..288004626 rank 3: 288004626..384006168
  27. # Computed per node by /home/kilork/aiden4x-launch.sh from $NODE_RANK
  28. # (the launcher exports NODE_RANK per rank; a full range on every rank would
  29. # double-count row contributions in the reduce, so the range is mandatory —
  30. # the overlay refuses a full snapshot without it).
  31. #
  32. # B12X_ROCE_GID_INDEX reuses the launcher's per-node NCCL GID auto-detect
  33. # (the wrapper exports it), so no pinned GID anywhere.
  34. engine = "sglang"
  35. name = "DeepSeek-V4.1-Flash (SGLang 4x, Aiden production-1.6)"
  36. docker_image = "aidendle94/sparkrun-sglang-dsv41-gb10:production-1.6"
  37. model = "deepseek-ai/DeepSeek-V4.1-Flash"
  38. parent_model = "deepseek-ai/DeepSeek-V4.1-Flash"
  39. port = 30000
  40. required_nodes = 4
  41. tensor_parallel_size = 4
  42. distributed_backend = "mp"
  43. shm_size = "32gb"
  44. # The wrapper (mounted read-only from the host via extra_mounts below) sets
  45. # SPARK_ENGRAM_ROWS (+ B12X_ROCE_GID_INDEX) from NODE_RANK and execs the
  46. # engine with the launcher's full flag list in "$@".
  47. entrypoint = ["sh", "/aiden4x-launch.sh"]
  48. args = [
  49. "--trust-remote-code",
  50. "--served-model-name", "deepseek-v4.1-flash",
  51. "--load-format", "safetensors",
  52. # Aiden's measured shape (production-1.6); --tp 4 / --nnodes 4 /
  53. # --node-rank / --dist-init-addr are appended by the launcher
  54. "--ep-size", "4",
  55. "--attention-backend", "dsv4",
  56. "--moe-runner-backend", "flashinfer_mxfp4",
  57. # required for the MXFP8 route (selects the block-scaled path the hook wraps;
  58. # K=576 projections fall back to CUTLASS per shape)
  59. "--fp8-gemm-backend", "flashinfer_cutlass",
  60. "--mem-fraction-static", "0.80",
  61. "--chunked-prefill-size", "2048",
  62. "--context-length", "524288",
  63. "--max-running-requests", "16",
  64. "--cuda-graph-max-bs-decode", "16",
  65. "--random-seed", "0",
  66. "--enable-decoder-swa-bounded-replay",
  67. "--speculative-algorithm", "DSPARK",
  68. "--speculative-dspark-block-size", "5",
  69. "--tool-call-parser", "deepseekv41",
  70. "--reasoning-parser", "deepseek-v41",
  71. "--watchdog-timeout", "1800",
  72. "--min-free-slots-delay", "1",
  73. ]
  74. # The launcher also appends: --model-path <resolved NAS snapshot>, --nnodes 4,
  75. # --node-rank N, --dist-init-addr <head-ib>:25000, --host 0.0.0.0, --port 30000.
  76. # NOTE: no --default-chat-template-kwargs '{"thinking":true}' — the bench
  77. # harness controls thinking; Aiden's default is for his interactive fleet.
  78. # The per-rank Engram range wrapper, mounted read-only (bare keys above; the
  79. # table header must come after them or `args`/`[env]` would parse into it).
  80. [[extra_mounts]]
  81. host = "/home/kilork/aiden4x-launch.sh"
  82. container = "/aiden4x-launch.sh"
  83. read_only = true
  84. # Engram row source: the full checkpoint snapshot on the NAS (container-side
  85. # path — the launcher mounts --hf-cache /mnt/nas/ai/huggingface at
  86. # /root/.cache/huggingface). No engram-local.json manifest in a full snapshot,
  87. # so per-rank ranges come from SPARK_ENGRAM_ROWS (wrapper).
  88. [env]
  89. SPARK_ENGRAM_DIR = "/root/.cache/huggingface/hub/models--deepseek-ai--DeepSeek-V4.1-Flash/snapshots/dba1be0a40aa45a94ad051997016db3960a90277"
  90. SPARK_ENGRAM_MODE = "hostnode"
  91. SPARK_ENGRAM_EARLY = "1"
  92. SPARK_ENGRAM_PREFETCH = "1"
  93. SPARK_PAGE_CACHE_RELEASE = "1"
  94. SPARK_LATE_TAIL_SKIP = "1"
  95. SPARK_LATE_TAIL_SKIP_ALL = "1"
  96. SPARK_SM120_REAL_HEADS = "1"
  97. SPARK_ENGRAM_THREADS = "64"
  98. SPARK_ENGRAM_MAX_IDS = "65536"
  99. SPARK_MXFP8_BACKEND = "b12x"
  100. SPARK_ROCE_AR = "1"
  101. SPARK_ROCE_AR_MAX = "1MB"
  102. SPARK_ROCE_AG_MAX = "16MB"
  103. # b12x RoCE runtime (GID index set by the wrapper from the detected value)
  104. B12X_ROCE_HCA = "rocep1s0f0,roceP2p1s0f0"
  105. B12X_ROCE_SPIN_LIMIT = "300000000"
  106. # image defaults Aiden overrides
  107. SGLANG_ENABLE_DSV41_ENGRAM_HOST_TABLE = "0"
  108. SGLANG_FLASHINFER_MOE_FUSED_FINALIZE = "0"
  109. SGLANG_DSV41_REASONING_EFFORT = "100"
  110. PYTORCH_CUDA_ALLOC_CONF = "expandable_segments:False"
  111. CUDA_DEVICE_ORDER = "PCI_BUS_ID"
  112. # Aiden's NCCL pins (4x 200G RoCE fabric, 192.168.200.0/24) — [env] wins over
  113. # the config's [nccl."4nodes-switch"] where they differ
  114. NCCL_NET = "IB"
  115. NCCL_IB_DISABLE = "0"
  116. NCCL_IB_HCA = "rocep1s0f0,roceP2p1s0f0"
  117. NCCL_SOCKET_IFNAME = "enp1s0f0np0"
  118. GLOO_SOCKET_IFNAME = "enp1s0f0np0"
  119. NCCL_IB_ROCE_VERSION_NUM = "2"
  120. NCCL_IB_ADDR_FAMILY = "AF_INET"
  121. NCCL_IB_ADDR_RANGE = "192.168.200.0/24"
  122. NCCL_IB_SUBNET_PREFIX_LEN = "24"
  123. NCCL_CROSS_NIC = "1"
  124. NCCL_IB_MERGE_NICS = "0"
  125. NCCL_IB_SUBNET_AWARE_ROUTING = "1"
  126. NCCL_CUMEM_ENABLE = "0"
  127. NCCL_P2P_DISABLE = "1"
  128. NCCL_SHM_DISABLE = "1"
  129. NCCL_BUFFSIZE = "1048576"
  130. NCCL_LL128_BUFFSIZE = "262144"
  131. NCCL_PROTO = "^LL128"
  132. NCCL_MAX_NCHANNELS = "8"