Accueil Explorer Aide
Connexion
sm121
/
launcher
2
0
Fork 0
Fichiers Tickets 0 Pull Requests 0 Wiki
Aborescence: ad5906d1a7
Branches Tags
launcher
/
recipes
/
deepseek-v4-flash
/
recipe.toml

recipe.toml 2.2 KB
Historique Raw

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465 
  1. engine = "vllm"
    2. 

  2. name = "DeepSeek V4 Flash (vLLM)"
    3. 

  3. docker_image = "aidendle94/sparkrun-vllm-ds4-gb10:production-v2"
    4. 

  4. model = "deepseek-ai/DeepSeek-V4-Flash"
    5. 

  5. required_nodes = 2
    6. 

  6. tensor_parallel_size = 2
    7. 

  7. pipeline_parallel_size = 1
    8. 

  8. distributed_backend = "mp"
    9. 

  9. 

  10. # --privileged makes --shm-size redundant; kept for documentation
    11. 

  11. # shm_size = "64gb"
    12. 

  12. 

  13. entrypoint = ["vllm", "serve"]
    14. 

  14. 

  15. args = [
    16. 

  16.     "--kv-cache-dtype", "fp8",
    17. 

  17.     "--block-size", "256",
    18. 

  18.     "--max-model-len", "524288",
    19. 

  19.     "--max-num-seqs", "8",
    20. 

  20.     "--max-num-batched-tokens", "8192",
    21. 

  21.     "--enable-chunked-prefill",
    22. 

  22.     "--gpu-memory-utilization", "0.85",
    23. 

  23.     "--compilation-config", """
    24. 

  24. '{"cudagraph_mode":"FULL_AND_PIECEWISE","custom_ops":["all"]}'""",
    25. 

  25.     "--enable-prefix-caching",
    26. 

  26.     "--load-format", "safetensors",
    27. 

  27.     "--speculative-config", """
    28. 

  28. '{"method":"mtp","num_speculative_tokens":2}'""",
    29. 

  29.     "--tokenizer-mode", "deepseek_v4",
    30. 

  30.     "--tool-call-parser", "deepseek_v4",
    31. 

  31.     "--enable-auto-tool-choice",
    32. 

  32.     "--reasoning-parser", "deepseek_v4",
    33. 

  33.     "--reasoning-config", """
    34. 

  34. '{"reasoning_parser":"deepseek_v4","reasoning_start_str":"","reasoning_end_str":""}'""",
    35. 

  35.     "--default-chat-template-kwargs", """
    36. 

  36. '{"thinking":true,"preserve_thinking":true}'""",
    37. 

  37.     "--default-chat-template-kwargs.reasoning_effort", "high",
    38. 

  38.     "--enable-flashinfer-autotune",
    39. 

  39.     "--trust-remote-code",
    40. 

  40.     "--served-model-name", "deepseek-v4-flash", "default",
    41. 

  41. ]
    42. 

  42. 

  43. [env]
    44. 

  44. # Ray memory management — prevents OOM during cudagraph compilation
    45. 

  45. RAY_memory_monitor_refresh_ms = "0"
    46. 

  46. RAY_num_prestart_python_workers = "0"
    47. 

  47. RAY_object_store_memory = "1073741824"
    48. 

  48. 

  49. # vLLM features for DeepSeek V4 on Blackwell (GB10 / sm_120)
    50. 

  50. VLLM_ALLOW_LONG_MAX_MODEL_LEN = "1"
    51. 

  51. VLLM_USE_B12X_MOE = "1"
    52. 

  52. VLLM_SPARSE_INDEXER_MAX_LOGITS_MB = "256"
    53. 

  53. 

  54. # Compile-time arch hints for PyTorch and FlashInfer on GB10
    55. 

  55. TORCH_CUDA_ARCH_LIST = "12.1a"
    56. 

  56. FLASHINFER_CUDA_ARCH_LIST = "12.1a"
    57. 

  57. 

  58. CUDA_HOME = "/opt/env/targets/sbsa-linux"
    59. 

  59. FLASHINFER_DISABLE_VERSION_CHECK = "1"
    60. 

  60. TILELANG_CLEANUP_TEMP_FILES = "1"
    61. 

  61. DG_JIT_USE_NVRTC = "0"
    62. 

  62. DG_JIT_PRINT_COMPILER_COMMAND = "1"
    63. 

  63. VLLM_NCCL_SO_PATH = "/opt/env/lib/python3.12/site-packages/nvidia/nccl/lib/libnccl.so.2"
    64. 

  64. HF_HUB_OFFLINE = "1"
    65. 

  65. TRANSFORMERS_OFFLINE = "1"
    66.