Domů Procházet Nápověda
Přihlásit se
sm121
/
recipes
2
0
Rozštěpit 0
Soubory Úkoly 0 Pull Requesty 0 Wiki
Strom: 9675392c36
Větve Značky
recipes
/
docker-images
/
glm53-kindling
/
experimental
/
compose
/
adaptive-k.yaml

adaptive-k.yaml 2.9 KB
Historie Surový

12345678910111213141516171819202122232425262728293031323334353637 
  1. # Adaptive speculation length: the scheduler picks how many of the 7 DFlash2
    2. 

  2. # drafts each step verifies (adaptive_k.py). The per-batch-size schedule only
    3. 

  3. # makes vLLM capture CUDA graphs for every level; the scheduler ignores it.
    4. 

  4. # draft-trunc/: after the drafts exist, each request drops its drafts whose
    5. 

  5. # predicted survival (vLLM's acceptance estimator) is below
    6. 

  6. # VLLM_DRAFT_TRUNC_TAU. Dead drafts keep the verify shape, skip their MoE
    7. 

  7. # experts and are rejected, so outputs are unchanged. Every
    8. 

  8. # VLLM_DRAFT_TRUNC_EXPLORE-th step skips the cut and trains the estimator.
    9. 

  9. # disable_eagle_block_drop: vLLM drops the last matching prefix-cache block for
    10. 

  10. # EAGLE-style drafters, 2,304 tokens here at TP=4, and every cache hit then
    11. 

  11. # recomputes it. With the drop off, an agent turn's time to first token falls by
    12. 

  12. # about a third and the output is identical (#54).
    13. 

  13. services:
    14. 

  14.   glm53:
    15. 

  15.     volumes:
    16. 

  16.       - ../experimental/adaptive-k/adaptive_k.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/core/sched/adaptive_k.py:ro
    17. 

  17.       - ../experimental/draft-trunc/draft_trunc.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu/spec_decode/draft_trunc.py:ro
    18. 

  18.       - ../experimental/draft-trunc/model_runner.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu/model_runner.py:ro
    19. 

  19.       - ../experimental/draft-trunc/rejection_sampler.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu/spec_decode/rejection_sampler.py:ro
    20. 

  20.       - ../experimental/draft-trunc/grouped_topk_router.py:/usr/local/lib/python3.12/dist-packages/vllm/model_executor/layers/fused_moe/router/grouped_topk_router.py:ro
    21. 

  21.       - ../experimental/draft-trunc/speculator.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu/spec_decode/speculator.py:ro
    22. 

  22.       - ../experimental/draft-trunc/dflash2_speculator.py:/usr/local/lib/python3.12/dist-packages/vllm/v1/worker/gpu/spec_decode/dflash2/speculator.py:ro
    23. 

  23.     environment:
    24. 

  24.       - EXTRA_ARGS=--scheduler-cls vllm.v1.core.sched.adaptive_k.AdaptiveKScheduler
    25. 

  25.       # Merged into the entrypoint's --speculative-config (method, drafter and
    26. 

  26.       # width stay SPEC_METHOD, DFLASH_MODEL and SPEC_TOKENS).
    27. 

  27.       - SPEC_EXTRA={"disable_eagle_block_drop":true,"attention_backend":"TRITON_ATTN","num_speculative_tokens_per_batch_size":[[1,1,7],[2,2,5],[3,3,4],[4,4,3],[5,5,2],[6,32,7]]}
    28. 

  28.       - VLLM_ADAPTIVE_K_CONTROL=/root/.cache/adaptive-k
    29. 

  29.       - VLLM_DRAFT_TRUNC_TAU=${VLLM_DRAFT_TRUNC_TAU:-0.3}
    30. 

  30.       - VLLM_DRAFT_TRUNC_EXPLORE=${VLLM_DRAFT_TRUNC_EXPLORE:-4}
    31. 

  31.       - VLLM_DRAFT_TRUNC_CONTROL=/root/.cache/draft-trunc
    32. 

  32.       - VLLM_DRAFT_TRUNC_LOG=${VLLM_DRAFT_TRUNC_LOG:-}
    33. 

  33.       - VLLM_ADAPTIVE_K_PER_REQUEST=${VLLM_ADAPTIVE_K_PER_REQUEST:-0}
    34. 

  34.       - VLLM_ADAPTIVE_K_CODE_MODE=${VLLM_ADAPTIVE_K_CODE_MODE:-gated}
    35. 

  35.       - VLLM_ADAPTIVE_K_ONLINE=${VLLM_ADAPTIVE_K_ONLINE:-1}
    36. 

  36.       - VLLM_ADAPTIVE_K_MODEL=${VLLM_ADAPTIVE_K_MODEL:-}
    37. 

  37.       - VLLM_ADAPTIVE_K_COST_MS=${VLLM_ADAPTIVE_K_COST_MS:-2:29.7,3:33.1,4:36.5,5:39.8,6:43.1,7:46.0,8:48.9}
    38.