40665.diff 49 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125
  1. diff --git a/docs/docs/advanced_features/server_arguments.mdx b/docs/docs/advanced_features/server_arguments.mdx
  2. index 608fe3166bfb..14bb5a3552fe 100644
  3. --- a/docs/docs/advanced_features/server_arguments.mdx
  4. +++ b/docs/docs/advanced_features/server_arguments.mdx
  5. @@ -1648,6 +1648,12 @@ Combining `--enable-response-store` with `--disaggregation-mode=prefill` or `dec
  6. <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`None`</td>
  7. <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Type: int</td>
  8. </tr>
  9. + <tr>
  10. + <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}>`--speculative-draft-kv-ratio`</td>
  11. + <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>The draft KV pool as a fraction of the target's token capacity. At 1.0 the draft holds one slot per target token. Below 1.0 the draft's KV lives in a sliding-window pool. Requires DFLASH, `--speculative-draft-window-size`, and a draft whose every layer is sliding_attention, and is not supported yet on targets with sliding-window layers of their own.</td>
  12. + <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}>`1.0`</td>
  13. + <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Type: float</td>
  14. + </tr>
  15. </tbody>
  16. </table>
  17. diff --git a/docs/docs/advanced_features/speculative_decoding.mdx b/docs/docs/advanced_features/speculative_decoding.mdx
  18. index 004e9ef75b81..ad67d65ac1b0 100644
  19. --- a/docs/docs/advanced_features/speculative_decoding.mdx
  20. +++ b/docs/docs/advanced_features/speculative_decoding.mdx
  21. @@ -565,6 +565,11 @@ Relevant parameters:
  22. <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>Draft KV sliding-window size. Must be <code>&gt;= speculative-num-draft-tokens</code> when set.</td>
  23. <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>None</code></td>
  24. </tr>
  25. + <tr>
  26. + <td style={{padding: "9px 12px", fontWeight: 500, backgroundColor: "rgba(255,255,255,0.02)"}}><code>--speculative-draft-kv-ratio</code></td>
  27. + <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.05)"}}>The draft KV pool as a fraction of the target's token capacity. Below 1.0 the draft's KV lives in a sliding-window pool and needs an all-SWA draft. Not supported yet on targets with sliding-window layers of their own.</td>
  28. + <td style={{padding: "9px 12px", backgroundColor: "rgba(255,255,255,0.02)"}}><code>1.0</code></td>
  29. + </tr>
  30. </tbody>
  31. </table>
  32. diff --git a/python/sglang/srt/arg_groups/field_order.py b/python/sglang/srt/arg_groups/field_order.py
  33. index 6b104eb4affb..930d47589b2e 100644
  34. --- a/python/sglang/srt/arg_groups/field_order.py
  35. +++ b/python/sglang/srt/arg_groups/field_order.py
  36. @@ -287,6 +287,7 @@
  37. "speculative_dsa_topk_backend",
  38. "speculative_draft_kv_cache_dtype",
  39. "speculative_draft_window_size",
  40. + "speculative_draft_kv_ratio",
  41. "speculative_moe_runner_backend",
  42. "speculative_moe_a2a_backend",
  43. "speculative_draft_model_quantization",
  44. diff --git a/python/sglang/srt/arg_groups/fields/spec.py b/python/sglang/srt/arg_groups/fields/spec.py
  45. index 5ae41ce085ed..e6ba4337d864 100644
  46. --- a/python/sglang/srt/arg_groups/fields/spec.py
  47. +++ b/python/sglang/srt/arg_groups/fields/spec.py
  48. @@ -166,6 +166,10 @@ class Spec(msgspec.Struct):
  49. Optional[int],
  50. "Number of leading 'attention sink' tokens the draft always attends to, in addition to the --speculative-draft-window-size recent window (StreamingLLM-style). Honored only by the built-in EAGLE/MTP draft-decode path on the Triton and FlashInfer draft backends; the Llama EAGLE-3 and DFLASH windows ignore it. 0/unset => pure recent window. Requires --speculative-draft-window-size.",
  51. ] = None
  52. + speculative_draft_kv_ratio: A[
  53. + float,
  54. + "The draft KV pool as a fraction of the target's token capacity. It should be between 0 and 1. At 1.0 (the default) the draft holds one slot per target token. Below 1.0 the draft's KV lives in a sliding-window pool whose out-of-window slots are freed and whose cached prefixes stay reusable while their trailing window survives. Requires DFLASH, --speculative-draft-window-size, and a draft whose every layer is sliding_attention, and is not supported yet on targets with sliding-window layers of their own. The pool must still cover the running requests, or startup fails with the smallest usable ratio.",
  55. + ] = 1.0
  56. speculative_moe_runner_backend: A[
  57. Optional[str],
  58. Arg(
  59. diff --git a/python/sglang/srt/arg_groups/speculative_hook.py b/python/sglang/srt/arg_groups/speculative_hook.py
  60. index 5afef4f83223..e50340a22862 100644
  61. --- a/python/sglang/srt/arg_groups/speculative_hook.py
  62. +++ b/python/sglang/srt/arg_groups/speculative_hook.py
  63. @@ -388,6 +388,16 @@ def _handle_dflash(server_args: ServerArgs) -> None:
  64. f"window_size={cfg.speculative_draft_window_size}, block_size={draft_tokens}."
  65. )
  66. + ratio = cfg.speculative_draft_kv_ratio
  67. + if not 0 < ratio <= 1:
  68. + raise ValueError(
  69. + "--speculative-draft-kv-ratio must be in (0, 1], got "
  70. + f"{ratio}. Above 1 the draft would hold slots the target cannot "
  71. + "address."
  72. + )
  73. + if ratio < 1:
  74. + _validate_dflash_draft_kv_ratio(server_args)
  75. +
  76. _resolve_dflash_draft_attention_backend(server_args)
  77. if cfg.max_running_requests is None:
  78. @@ -401,6 +411,53 @@ def _handle_dflash(server_args: ServerArgs) -> None:
  79. )
  80. +def _validate_dflash_draft_kv_ratio(server_args: ServerArgs) -> None:
  81. + """We free a request's draft slots as they leave the window, so every draft
  82. + layer must be sliding-window attention."""
  83. + cfg = resolving_view(server_args)
  84. + if cfg.speculative_draft_window_size is None:
  85. + raise ValueError(
  86. + "--speculative-draft-kv-ratio below 1 requires "
  87. + "--speculative-draft-window-size."
  88. + )
  89. + if not cfg.device.startswith("cuda"):
  90. + raise ValueError(
  91. + "--speculative-draft-kv-ratio below 1 is only supported on CUDA."
  92. + )
  93. + unsupported = [
  94. + flag
  95. + for flag, enabled in (
  96. + ("--enable-hierarchical-cache", cfg.enable_hierarchical_cache),
  97. + ("--enable-unified-memory", cfg.enable_unified_memory),
  98. + ("--enable-hisparse", cfg.enable_hisparse),
  99. + ("--disaggregation-mode", cfg.disaggregation_mode != "null"),
  100. + ("--dcp-size", cfg.dcp_size > 1),
  101. + )
  102. + if enabled
  103. + ]
  104. + if unsupported:
  105. + raise ValueError(
  106. + "--speculative-draft-kv-ratio below 1 does not support "
  107. + f"{', '.join(unsupported)}."
  108. + )
  109. +
  110. + from sglang.srt.speculative.dflash_utils import get_dflash_layer_types
  111. + from sglang.srt.utils.hf_transformers_utils import get_config
  112. +
  113. + draft_hf_config = get_config(
  114. + cfg.speculative_draft_model_path,
  115. + trust_remote_code=cfg.trust_remote_code,
  116. + revision=cfg.speculative_draft_model_revision,
  117. + model_override_args=json.loads(cfg.json_model_override_args),
  118. + )
  119. + layer_types = get_dflash_layer_types(draft_hf_config)
  120. + if not layer_types or any(t != "sliding_attention" for t in layer_types):
  121. + raise ValueError(
  122. + "--speculative-draft-kv-ratio below 1 requires every draft layer to "
  123. + f"be sliding_attention, got layer_types={layer_types}."
  124. + )
  125. +
  126. +
  127. def _handle_uno(server_args: ServerArgs) -> None:
  128. cfg = resolving_view(server_args)
  129. diff --git a/python/sglang/srt/mem_cache/allocator/swa.py b/python/sglang/srt/mem_cache/allocator/swa.py
  130. index cee82b123606..9ebb23f9815b 100644
  131. --- a/python/sglang/srt/mem_cache/allocator/swa.py
  132. +++ b/python/sglang/srt/mem_cache/allocator/swa.py
  133. @@ -1,4 +1,5 @@
  134. import logging
  135. +from typing import Optional
  136. import torch
  137. @@ -7,6 +8,7 @@
  138. from sglang.srt.mem_cache.allocator.paged import PagedTokenToKVPoolAllocator
  139. from sglang.srt.mem_cache.allocator.token import TokenToKVPoolAllocator
  140. from sglang.srt.mem_cache.base_swa_memory_pool import BaseSWAKVPool
  141. +from sglang.srt.mem_cache.memory_pool import KVCache
  142. from sglang.srt.utils import is_npu
  143. from sglang.srt.utils.common import get_num_new_pages
  144. from sglang.srt.utils.invariants import Bucket, Invariant, IsTrue, expect
  145. @@ -866,3 +868,82 @@ def clear(self):
  146. def is_swa_req_ring(allocator) -> bool:
  147. return isinstance(allocator, SWATokenToKVPoolAllocator) and allocator.swa_req_ring
  148. +
  149. +
  150. +class _DraftSWAKVPoolBinding(BaseSWAKVPool):
  151. + """The target's pool with a speculative draft's pool as the SWA side.
  152. +
  153. + The draft runner builds its pool after the target allocator, so the SWA side
  154. + starts out None and takes the mapping when it is attached.
  155. + """
  156. +
  157. + swa_req_ring_size = None
  158. +
  159. + def __init__(self, full_kv_pool: KVCache):
  160. + self.full_kv_pool = full_kv_pool
  161. + self.swa_kv_pool: Optional[BaseSWAKVPool] = None
  162. + self.full_to_swa_index_mapping: Optional[torch.Tensor] = None
  163. +
  164. + def attach_swa_kv_pool(self, swa_kv_pool: BaseSWAKVPool) -> None:
  165. + self.swa_kv_pool = swa_kv_pool
  166. + swa_kv_pool.register_mapping(self.full_to_swa_index_mapping)
  167. +
  168. + def register_mapping(self, full_to_swa_index_mapping: torch.Tensor) -> None:
  169. + self.full_to_swa_index_mapping = full_to_swa_index_mapping
  170. + if self.swa_kv_pool is not None:
  171. + self.swa_kv_pool.register_mapping(full_to_swa_index_mapping)
  172. +
  173. + def translate_loc_from_full_to_swa(self, kv_indices: torch.Tensor) -> torch.Tensor:
  174. + return self.full_to_swa_index_mapping[kv_indices]
  175. +
  176. + def get_state_buf_infos(self):
  177. + return self.swa_kv_pool.get_contiguous_buf_infos()
  178. +
  179. + def get_key_buffer(self, layer_id: int):
  180. + return self.full_kv_pool.get_key_buffer(layer_id)
  181. +
  182. + def get_value_buffer(self, layer_id: int):
  183. + return self.full_kv_pool.get_value_buffer(layer_id)
  184. +
  185. + def get_kv_buffer(self, layer_id: int):
  186. + return self.full_kv_pool.get_kv_buffer(layer_id)
  187. +
  188. + def set_kv_buffer(self, *args, **kwargs):
  189. + return self.full_kv_pool.set_kv_buffer(*args, **kwargs)
  190. +
  191. +
  192. +class DraftSWATokenToKVPoolAllocator(SWATokenToKVPoolAllocator):
  193. + """Target allocator whose SWA side is a bounded speculative draft KV pool.
  194. +
  195. + Every target layer is full attention. The SWA slot handed out with each
  196. + target slot indexes the draft pool.
  197. + """
  198. +
  199. + def __init__(
  200. + self,
  201. + size: int,
  202. + size_swa: int,
  203. + page_size: int,
  204. + dtype: torch.dtype,
  205. + device: str,
  206. + kvcache: KVCache,
  207. + need_sort: bool,
  208. + req_to_token_pool=None,
  209. + ):
  210. + super().__init__(
  211. + size,
  212. + size_swa,
  213. + page_size,
  214. + dtype,
  215. + device,
  216. + _DraftSWAKVPoolBinding(kvcache),
  217. + need_sort,
  218. + req_to_token_pool=req_to_token_pool,
  219. + )
  220. +
  221. + def attach_draft_kv_pool(self, draft_kv_pool: BaseSWAKVPool) -> None:
  222. + self._kvcache.attach_swa_kv_pool(draft_kv_pool)
  223. +
  224. + def get_kvcache(self):
  225. + # Backends check this to detect a hybrid-SWA target.
  226. + return self._kvcache.full_kv_pool
  227. diff --git a/python/sglang/srt/mem_cache/kv_cache_builder.py b/python/sglang/srt/mem_cache/kv_cache_builder.py
  228. index f403e7cd43b6..42ac470a94dd 100644
  229. --- a/python/sglang/srt/mem_cache/kv_cache_builder.py
  230. +++ b/python/sglang/srt/mem_cache/kv_cache_builder.py
  231. @@ -55,6 +55,7 @@ class KVCacheBuildResult:
  232. get_memory,
  233. get_parallel,
  234. get_schedule,
  235. + get_spec,
  236. )
  237. from sglang.srt.speculative.base_spec_worker import HiCacheDraftMode
  238. from sglang.srt.utils import is_hip
  239. @@ -277,16 +278,26 @@ def build_kv_cache(
  240. # Hybrid memory pool
  241. token_to_kv_pool = tp_worker.model_runner.token_to_kv_pool
  242. - is_hybrid_swa = tp_worker.is_hybrid_swa and (
  243. - not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
  244. - or token_to_kv_pool.needs_paged_swa_allocator
  245. + # The bounded draft's KV lives on the SWA side of the allocator, so the
  246. + # cache maintains it like a hybrid-SWA pool.
  247. + bounded_draft_kv = tp_worker.model_runner.draft_kv_ratio < 1
  248. + is_hybrid_swa = bounded_draft_kv or (
  249. + tp_worker.is_hybrid_swa
  250. + and (
  251. + not isinstance(token_to_kv_pool, DeepSeekV4TokenToKVPool)
  252. + or token_to_kv_pool.needs_paged_swa_allocator
  253. + )
  254. )
  255. is_hybrid_ssm = uses_ssm_state(tp_worker.model_runner.model_config)
  256. is_dsa = is_deepseek_dsa(model_config.hf_config)
  257. sliding_window_size = None
  258. if is_hybrid_swa:
  259. - sliding_window_size = tp_worker.sliding_window_size
  260. + sliding_window_size = (
  261. + get_spec().speculative_draft_window_size
  262. + if bounded_draft_kv
  263. + else tp_worker.sliding_window_size
  264. + )
  265. full_tokens_per_layer, swa_tokens_per_layer = (
  266. tp_worker.get_tokens_per_layer_info()
  267. )
  268. diff --git a/python/sglang/srt/mem_cache/kv_cache_configurator.py b/python/sglang/srt/mem_cache/kv_cache_configurator.py
  269. index 97382225b680..08d434955ca8 100644
  270. --- a/python/sglang/srt/mem_cache/kv_cache_configurator.py
  271. +++ b/python/sglang/srt/mem_cache/kv_cache_configurator.py
  272. @@ -45,6 +45,7 @@
  273. HiSparseTokenToKVPoolAllocator,
  274. )
  275. from sglang.srt.mem_cache.allocator.swa import (
  276. + DraftSWATokenToKVPoolAllocator,
  277. PureSWATokenToKVPoolAllocator,
  278. SWATokenToKVPoolAllocator,
  279. is_swa_req_ring,
  280. @@ -289,10 +290,31 @@ class KVCacheConfigurator:
  281. hybrid_kda_config: Optional[Any] = field(init=False)
  282. is_hybrid_swa_mtp_draft: bool = field(init=False)
  283. draft_swa_full_capacity: bool = field(init=False)
  284. + draft_kv_ratio: float = field(init=False)
  285. def __post_init__(self) -> None:
  286. self.mambaish_config = mambaish_config(self.model_config)
  287. self.hybrid_gdn_config = hybrid_gdn_config(self.model_config)
  288. + # Below 1 the draft's KV moves to the SWA side of the target allocator.
  289. + if self.is_draft_worker:
  290. + self.draft_kv_ratio = (
  291. + get_spec().speculative_draft_kv_ratio
  292. + if isinstance(
  293. + self.token_to_kv_pool_allocator, DraftSWATokenToKVPoolAllocator
  294. + )
  295. + else 1.0
  296. + )
  297. + else:
  298. + self.draft_kv_ratio = (
  299. + get_spec().speculative_draft_kv_ratio
  300. + if self.spec_algorithm.is_dflash_family()
  301. + else 1.0
  302. + )
  303. + if self.draft_kv_ratio < 1 and self.is_hybrid_swa:
  304. + raise ValueError(
  305. + "--speculative-draft-kv-ratio below 1 is not supported yet "
  306. + "on a target with sliding-window layers."
  307. + )
  308. self.hybrid_kda_config = hybrid_kda_config(self.model_config)
  309. self.is_hybrid_swa_mtp_draft = (
  310. self.is_draft_worker
  311. @@ -419,7 +441,7 @@ def _derive_pool_sizes(self, *, config: MemoryPoolConfig) -> _PoolSizes:
  312. max_running_requests = config.max_running_requests
  313. full_max_total_num_tokens = None
  314. swa_max_total_num_tokens = None
  315. - if self.is_hybrid_swa:
  316. + if self.is_hybrid_swa or self.draft_kv_ratio < 1:
  317. full_max_total_num_tokens = config.full_max_total_num_tokens
  318. swa_max_total_num_tokens = config.swa_max_total_num_tokens
  319. @@ -1302,6 +1324,12 @@ def _build_token_to_kv_pool(
  320. swa_max_total_num_tokens=sizes.swa_max_total_num_tokens,
  321. mha_pool_class=mha_pool_class,
  322. )
  323. + elif self.is_draft_worker and self.draft_kv_ratio < 1:
  324. + token_to_kv_pool = self._build_draft_swa_kv_pool(
  325. + full_max_total_num_tokens=sizes.full_max_total_num_tokens,
  326. + swa_max_total_num_tokens=sizes.swa_max_total_num_tokens,
  327. + mha_pool_class=mha_pool_class,
  328. + )
  329. elif is_minimax_sparse(self.model_config.hf_config):
  330. token_to_kv_pool = self._build_minimax_sparse_kv_pool(
  331. max_total_num_tokens=sizes.max_total_num_tokens,
  332. @@ -1827,6 +1855,38 @@ def _build_hybrid_swa_kv_pool(
  333. )
  334. return token_to_kv_pool
  335. + def _build_draft_swa_kv_pool(
  336. + self,
  337. + *,
  338. + full_max_total_num_tokens: int,
  339. + swa_max_total_num_tokens: int,
  340. + mha_pool_class: type,
  341. + ) -> KVCache:
  342. + """Draft pool addressed through the target allocator's full->SWA mapping.
  343. +
  344. + Every draft layer sits on the SWA side. The full side holds no layers
  345. + and only spans the target's token space.
  346. + """
  347. + return SWAKVPool(
  348. + size=full_max_total_num_tokens,
  349. + size_swa=swa_max_total_num_tokens,
  350. + page_size=self.pool_page_size,
  351. + dtype=self.kv_cache_dtype,
  352. + post_capture_active=self.post_capture_kv_active,
  353. + head_num=self.model_config.get_num_kv_heads(
  354. + get_parallel().attn_tp_size, get_parallel().attn_dcp_size
  355. + ),
  356. + head_dim=self.model_config.head_dim,
  357. + v_head_dim=self.model_config.v_head_dim,
  358. + swa_attention_layer_ids=list(
  359. + range(self.layer_info.start_layer, self.layer_info.end_layer)
  360. + ),
  361. + full_attention_layer_ids=[],
  362. + device=self.device,
  363. + enable_kv_cache_copy=True,
  364. + token_to_kv_pool_class=mha_pool_class,
  365. + )
  366. +
  367. def _build_minimax_sparse_kv_pool(self, *, max_total_num_tokens: int) -> KVCache:
  368. from sglang.srt.server_args import m3_fp8_attn_gemm_enabled
  369. @@ -2114,6 +2174,17 @@ def _build_token_to_kv_pool_allocator(
  370. need_sort=need_sort,
  371. req_to_token_pool=req_to_token_pool,
  372. )
  373. + elif self.draft_kv_ratio < 1:
  374. + token_to_kv_pool_allocator = DraftSWATokenToKVPoolAllocator(
  375. + sizes.full_max_total_num_tokens,
  376. + sizes.swa_max_total_num_tokens,
  377. + page_size=get_schedule().page_size,
  378. + dtype=self.kv_cache_dtype,
  379. + device=self.device,
  380. + kvcache=token_to_kv_pool,
  381. + need_sort=need_sort,
  382. + req_to_token_pool=req_to_token_pool,
  383. + )
  384. else:
  385. if get_memory().enable_hisparse:
  386. from sglang.srt.mem_cache.sparsity import (
  387. @@ -2199,6 +2270,8 @@ def _build_token_to_kv_pool_allocator(
  388. token_to_kv_pool.register_mapping(
  389. swa_allocator.full_to_swa_index_mapping
  390. )
  391. + elif self.draft_kv_ratio < 1:
  392. + token_to_kv_pool_allocator.attach_draft_kv_pool(token_to_kv_pool)
  393. return token_to_kv_pool_allocator
  394. def _profile_available_bytes(self, pre_model_load_memory: int) -> int:
  395. diff --git a/python/sglang/srt/mem_cache/swa_memory_pool.py b/python/sglang/srt/mem_cache/swa_memory_pool.py
  396. index 1206168478dd..0efdb69b3e91 100644
  397. --- a/python/sglang/srt/mem_cache/swa_memory_pool.py
  398. +++ b/python/sglang/srt/mem_cache/swa_memory_pool.py
  399. @@ -277,6 +277,33 @@ def set_kv_buffer(
  400. layer_id_override=layer_id_pool,
  401. )
  402. + def set_kv_buffer_prefix_valid(
  403. + self,
  404. + layer: RadixAttention,
  405. + loc_info,
  406. + commit_lens: torch.Tensor,
  407. + cache_k: torch.Tensor,
  408. + cache_v: torch.Tensor,
  409. + k_scale: Optional[float] = None,
  410. + v_scale: Optional[float] = None,
  411. + ):
  412. + loc_2d, swa_loc_2d, _ = unwrap_write_loc(loc_info)
  413. + layer_id_pool, is_swa_layer = self.layers_mapping[layer.layer_id]
  414. + pool = self.swa_kv_pool if is_swa_layer else self.full_kv_pool
  415. + if is_swa_layer:
  416. + assert swa_loc_2d is not None
  417. + loc_2d = swa_loc_2d
  418. + pool.set_kv_buffer_prefix_valid(
  419. + None,
  420. + loc_2d,
  421. + commit_lens,
  422. + cache_k,
  423. + cache_v,
  424. + k_scale,
  425. + v_scale,
  426. + layer_id_override=layer_id_pool,
  427. + )
  428. +
  429. def set_mla_kv_buffer(
  430. self,
  431. layer: RadixAttention,
  432. diff --git a/python/sglang/srt/model_executor/model_runner.py b/python/sglang/srt/model_executor/model_runner.py
  433. index 9f7a69c7b87f..423b5b1a6bb9 100644
  434. --- a/python/sglang/srt/model_executor/model_runner.py
  435. +++ b/python/sglang/srt/model_executor/model_runner.py
  436. @@ -369,6 +369,9 @@ def __init__(
  437. self.mtp_draft_device_pools = ()
  438. self.is_hybrid_swa = model_config.is_hybrid_swa
  439. self.is_hybrid_swa_compress = model_config.is_hybrid_swa_compress
  440. + # Below 1 the DFLASH draft's KV is the SWA side of the allocator.
  441. + # See KVCacheConfigurator.draft_kv_ratio.
  442. + self.draft_kv_ratio = 1.0
  443. self.use_mla_backend = self.model_config.attention_arch == AttentionArch.MLA
  444. self.attention_chunk_size = model_config.attention_chunk_size
  445. self.enable_elastic_ep = get_exec().moe.elastic_ep_backend is not None
  446. @@ -907,7 +910,8 @@ def alloc_memory_pool(self, memory_pool_config: Optional[MemoryPoolConfig] = Non
  447. self.token_to_kv_pool = result.token_to_kv_pool
  448. self.token_to_kv_pool_allocator = result.token_to_kv_pool_allocator
  449. self.memory_pool_config = result.memory_pool_config
  450. - if self.is_hybrid_swa:
  451. + self.draft_kv_ratio = self.kv_cache_configurator.draft_kv_ratio
  452. + if self.is_hybrid_swa or self.draft_kv_ratio < 1:
  453. self.full_max_total_num_tokens = result.full_max_total_num_tokens
  454. self.swa_max_total_num_tokens = result.swa_max_total_num_tokens
  455. # Keep a reference so the shared byte buffer is not GC'd.
  456. diff --git a/python/sglang/srt/model_executor/pool_configurator.py b/python/sglang/srt/model_executor/pool_configurator.py
  457. index 49304cb48e14..9ee9c058281d 100644
  458. --- a/python/sglang/srt/model_executor/pool_configurator.py
  459. +++ b/python/sglang/srt/model_executor/pool_configurator.py
  460. @@ -15,7 +15,7 @@
  461. import logging
  462. from bisect import bisect_right
  463. -from dataclasses import dataclass
  464. +from dataclasses import dataclass, replace
  465. from typing import TYPE_CHECKING, Optional
  466. import torch
  467. @@ -119,6 +119,12 @@ def _dflash_draft_cell_size(kvc: KVCacheConfigurator) -> int:
  468. return int(cell_size) * get_parallel().attn_dcp_size
  469. +def _draft_swa_cell_size(kvc: KVCacheConfigurator) -> int:
  470. + """Bytes/token of the draft KV pool at ``draft_kv_ratio`` slots per target
  471. + token, floored at 1 byte for ratios that truncate to 0."""
  472. + return max(1, int(_dflash_draft_cell_size(kvc) * kvc.draft_kv_ratio))
  473. +
  474. +
  475. def _get_dsa_cache_layer_ids(kvc: KVCacheConfigurator, num_layers: int) -> list[int]:
  476. """Global layer ids represented by the local DSA pool's dense layer slots."""
  477. if kvc.mambaish_config and not kvc.is_draft_worker:
  478. @@ -285,12 +291,15 @@ def __init__(self, kvc: KVCacheConfigurator):
  479. and int(draft_num_layers) > 0
  480. and int(num_layers) > 0
  481. ):
  482. + draft_cell_size = _dflash_draft_cell_size(kvc) or None
  483. + if draft_cell_size is not None and kvc.draft_kv_ratio < 1:
  484. + draft_cell_size = _draft_swa_cell_size(kvc)
  485. self._cell_size = scale_kv_cell_size_per_token_for_dflash(
  486. target_cell_size_per_token=self._cell_size,
  487. target_num_layers=int(num_layers),
  488. draft_num_layers=int(draft_num_layers)
  489. * get_parallel().attn_dcp_size,
  490. - draft_cell_size_per_token=_dflash_draft_cell_size(kvc) or None,
  491. + draft_cell_size_per_token=draft_cell_size,
  492. )
  493. def _compute_cell_size(self, kvc: KVCacheConfigurator, num_layers: int) -> int:
  494. @@ -937,6 +946,94 @@ def calculate_pool_sizes_from_max_tokens(
  495. )
  496. +class DFlashDraftSWAPoolConfigurator(DefaultPoolConfigurator):
  497. + """Sizes the draft KV pool at ``draft_kv_ratio`` slots per target token."""
  498. +
  499. + def __init__(self, kvc: KVCacheConfigurator):
  500. + super().__init__(kvc)
  501. + self._draft_kv_ratio = kvc.draft_kv_ratio
  502. + self._page_size = kvc.page_size
  503. + self._window = get_spec().speculative_draft_window_size
  504. + self._draft_cell_size = _dflash_draft_cell_size(kvc)
  505. + if self._draft_cell_size <= 0:
  506. + # _dflash_draft_cell_size is 0 when the draft config could not be
  507. + # read, and the layer-count fallback below ignores the ratio.
  508. + raise ValueError(
  509. + "--speculative-draft-kv-ratio below 1 needs the "
  510. + "DFLASH draft's KV bytes/token, which could not be resolved "
  511. + "from the draft config (see the earlier warning)."
  512. + )
  513. + self._target_cell_size = self._cell_size - _draft_swa_cell_size(kvc)
  514. + self._draft_cap = compute_swa_request_cap(
  515. + page_size=kvc.page_size,
  516. + window=self._window,
  517. + attn_dp_size=kvc.attn_dp_size,
  518. + )
  519. + # A mamba target resumes a hit at the last tracked state, up to
  520. + # mamba_track_interval before the request end, and a cached prefix keeps
  521. + # its window back from there.
  522. + self._cached_prefix_cost = self._window + kvc.page_size
  523. + if mambaish_config(kvc.model_config) is not None:
  524. + self._cached_prefix_cost += get_exec().mamba.mamba_track_interval
  525. +
  526. + def _with_draft_pool(
  527. + self, config: MemoryPoolConfig, min_ratio: float
  528. + ) -> MemoryPoolConfig:
  529. + target_tokens = config.max_total_num_tokens
  530. + draft_tokens = (
  531. + int(target_tokens * self._draft_kv_ratio) // self._page_size
  532. + ) * self._page_size
  533. + if draft_tokens < self._draft_cap:
  534. + raise RuntimeError(
  535. + f"--speculative-draft-kv-ratio {self._draft_kv_ratio} gives the "
  536. + f"draft {draft_tokens} slots, below the {self._draft_cap} its "
  537. + f"running requests need. Use at least {min_ratio}, or lower "
  538. + f"--max-running-requests or SGLANG_SWA_EVICTION_INTERVAL."
  539. + )
  540. + logger.info(
  541. + f"DFLASH draft KV pool: window={self._window}, "
  542. + f"ratio={self._draft_kv_ratio}, tokens={draft_tokens} "
  543. + f"(request cap {self._draft_cap}, "
  544. + f"{(draft_tokens - self._draft_cap) // self._cached_prefix_cost} "
  545. + f"cached prefixes at max concurrency)"
  546. + )
  547. + return replace(
  548. + config,
  549. + full_max_total_num_tokens=target_tokens,
  550. + swa_max_total_num_tokens=draft_tokens,
  551. + )
  552. +
  553. + def calculate_pool_sizes(
  554. + self, available_bytes: int, page_size: int
  555. + ) -> MemoryPoolConfig:
  556. + config = super().calculate_pool_sizes(available_bytes, page_size)
  557. + # budget = target_tokens * (target_cell + ratio * draft_cell), so the
  558. + # smallest ratio covering the cap is
  559. + # cap * target_cell / (budget - cap * draft_cell).
  560. + budget = config.max_total_num_tokens * self._cell_size
  561. + return self._with_draft_pool(
  562. + config,
  563. + ceil_div(
  564. + self._draft_cap * self._target_cell_size * 1000,
  565. + max(1, budget - self._draft_cap * self._draft_cell_size),
  566. + )
  567. + / 1000,
  568. + )
  569. +
  570. + def calculate_pool_sizes_from_max_tokens(
  571. + self, max_total_num_tokens: int, page_size: int
  572. + ) -> MemoryPoolConfig:
  573. + config = super().calculate_pool_sizes_from_max_tokens(
  574. + max_total_num_tokens, page_size
  575. + )
  576. + # A token cap fixes the target pool, so the ratio alone sets the draft's.
  577. + return self._with_draft_pool(
  578. + config,
  579. + ceil_div(self._draft_cap * 1000, max(1, config.max_total_num_tokens))
  580. + / 1000,
  581. + )
  582. +
  583. +
  584. # Used when --swa-full-tokens-ratio is at its default and cap mode is unusable.
  585. DSV4_DEFAULT_SWA_FULL_TOKENS_RATIO = 0.1
  586. @@ -1539,5 +1636,9 @@ def create_memory_pool_configurator(
  587. if SWAChunkCapPoolConfigurator.is_applicable(kvc):
  588. return SWAChunkCapPoolConfigurator(kvc)
  589. return HybridSWAPoolConfigurator(kvc)
  590. + # Future: targets with sliding-window layers of their own, which need a
  591. + # second sliding-window component for the draft.
  592. + if kvc.draft_kv_ratio < 1:
  593. + return DFlashDraftSWAPoolConfigurator(kvc)
  594. # Future: MambaPoolConfigurator
  595. return DefaultPoolConfigurator(kvc)
  596. diff --git a/python/sglang/srt/speculative/dflash_worker_v2.py b/python/sglang/srt/speculative/dflash_worker_v2.py
  597. index c55007f3df14..bb757c07be59 100644
  598. --- a/python/sglang/srt/speculative/dflash_worker_v2.py
  599. +++ b/python/sglang/srt/speculative/dflash_worker_v2.py
  600. @@ -25,6 +25,8 @@
  601. from sglang.srt.managers.schedule_batch import ScheduleBatch
  602. from sglang.srt.managers.scheduler import GenerationBatchResult
  603. from sglang.srt.managers.tp_worker import TpModelWorker
  604. +from sglang.srt.mem_cache.memory_pool import KVWriteLoc
  605. +from sglang.srt.mem_cache.swa_memory_pool import SWAKVPool
  606. from sglang.srt.model_executor.cuda_graph_config import Backend
  607. from sglang.srt.model_executor.forward_batch_info import (
  608. CaptureHiddenMode,
  609. @@ -1722,6 +1724,15 @@ def _ensure_local_reduce_buffers(
  610. return out_tokens
  611. + def _draft_kv_write_loc(self, cache_loc: torch.Tensor):
  612. + """Add the draft-side loc for a bounded draft pool."""
  613. + pool = self.draft_model_runner.token_to_kv_pool
  614. + if not isinstance(pool, SWAKVPool):
  615. + return cache_loc
  616. + return KVWriteLoc(
  617. + loc=cache_loc, swa_loc=pool.translate_loc_from_full_to_swa(cache_loc)
  618. + )
  619. +
  620. def _append_target_hidden_to_draft_kv_by_loc(
  621. self,
  622. *,
  623. @@ -1857,6 +1868,7 @@ def _append_target_hidden_to_draft_kv_by_loc(
  624. self._use_fused_kv_materialize = False
  625. self._fused_kv_helper = None
  626. + write_loc_2d = self._draft_kv_write_loc(cache_loc_2d)
  627. for layer in self.draft_model.layers:
  628. attn = layer.self_attn
  629. layer_ctx_hidden = self.draft_model.prepare_context_hidden_for_kv(
  630. @@ -1870,7 +1882,7 @@ def _append_target_hidden_to_draft_kv_by_loc(
  631. self.draft_model_runner.token_to_kv_pool.set_kv_buffer_prefix_valid(
  632. attn.attn,
  633. - cache_loc_2d,
  634. + write_loc_2d,
  635. commit_lens,
  636. k,
  637. v,
  638. @@ -1907,6 +1919,7 @@ def _append_target_hidden_sequential(
  639. ctx_positions: torch.Tensor,
  640. ctx_cache_loc: torch.Tensor,
  641. ) -> None:
  642. + write_loc = self._draft_kv_write_loc(ctx_cache_loc)
  643. for layer in self.draft_model.layers:
  644. attn = layer.self_attn
  645. layer_ctx_hidden = self.draft_model.prepare_context_hidden_for_kv(
  646. @@ -1925,7 +1938,7 @@ def _append_target_hidden_sequential(
  647. v = v.view(-1, attn.num_kv_heads, attn.head_dim)
  648. self.draft_model_runner.token_to_kv_pool.set_kv_buffer(
  649. attn.attn,
  650. - ctx_cache_loc,
  651. + write_loc,
  652. k,
  653. v,
  654. attn.attn.k_scale,
  655. @@ -1944,6 +1957,12 @@ def _append_target_hidden_fused(
  656. token_to_kv_pool = self.draft_model_runner.token_to_kv_pool
  657. if self._fused_kv_helper is None:
  658. raise RuntimeError("DFLASH fused KV helper is not initialized.")
  659. + write_loc = self._draft_kv_write_loc(ctx_cache_loc)
  660. + write_loc_2d = (
  661. + None
  662. + if ctx_cache_loc_2d is None
  663. + else self._draft_kv_write_loc(ctx_cache_loc_2d)
  664. + )
  665. def _write_layer_kv(
  666. layer_idx: int,
  667. @@ -1951,10 +1970,10 @@ def _write_layer_kv(
  668. cache_v: torch.Tensor,
  669. ) -> None:
  670. attn = self.draft_model.layers[layer_idx].self_attn.attn
  671. - if ctx_cache_loc_2d is not None and commit_lens is not None:
  672. + if write_loc_2d is not None and commit_lens is not None:
  673. token_to_kv_pool.set_kv_buffer_prefix_valid(
  674. attn,
  675. - ctx_cache_loc_2d,
  676. + write_loc_2d,
  677. commit_lens,
  678. cache_k,
  679. cache_v,
  680. @@ -1964,7 +1983,7 @@ def _write_layer_kv(
  681. else:
  682. token_to_kv_pool.set_kv_buffer(
  683. attn,
  684. - ctx_cache_loc,
  685. + write_loc,
  686. cache_k,
  687. cache_v,
  688. attn.k_scale,
  689. diff --git a/test/registered/spec/dflash/test_dflash_draft_kv_ratio.py b/test/registered/spec/dflash/test_dflash_draft_kv_ratio.py
  690. new file mode 100644
  691. index 000000000000..1d2614a12d3d
  692. --- /dev/null
  693. +++ b/test/registered/spec/dflash/test_dflash_draft_kv_ratio.py
  694. @@ -0,0 +1,219 @@
  695. +"""End-to-end test for --speculative-draft-kv-ratio.
  696. +
  697. +Pairs a linear/full-attention target with an all-SWA DFLASH draft:
  698. +``Qwen/Qwen3.8-27B`` and ``z-lab/Qwen3.8-27B-DFlash2`` (5 ``sliding_attention``
  699. +layers, ``sliding_window`` 2048, block size 8). Below 1.0 the draft's KV holds
  700. +a fraction of the target's tokens in a sliding-window pool instead of one slot
  701. +per target token, and a radix prefix hit still reuses the whole cached prefix.
  702. +"""
  703. +
  704. +import os
  705. +import re
  706. +import tempfile
  707. +import unittest
  708. +from contextlib import ExitStack
  709. +
  710. +import requests
  711. +
  712. +from sglang.srt.environ import envs
  713. +from sglang.srt.utils import kill_process_tree
  714. +from sglang.test.ci.ci_register import register_cuda_ci
  715. +from sglang.test.kits.eval_accuracy_kit import GSM8KMixin
  716. +from sglang.test.test_utils import (
  717. + DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH,
  718. + DEFAULT_URL_FOR_TEST,
  719. + CustomTestCase,
  720. + popen_launch_server,
  721. +)
  722. +
  723. +register_cuda_ci(est_time=500, stage="nightly", runner_config="1-gpu-large")
  724. +
  725. +TARGET_MODEL = "Qwen/Qwen3.8-27B"
  726. +DRAFT_MODEL = "z-lab/Qwen3.8-27B-DFlash2"
  727. +DRAFT_WINDOW = 2048
  728. +DRAFT_BLOCK_SIZE = 8
  729. +DRAFT_KV_RATIO = 0.25
  730. +MAX_RUNNING_REQUESTS = 8
  731. +# The target resumes a prefix hit at its last linear-attention checkpoint,
  732. +# which is chunk aligned; allow one chunk of slack before the prompt end.
  733. +CHECKPOINT_SLACK = 128
  734. +
  735. +# Several draft windows long, so a repeated request is a prefix hit whose
  736. +# draft window lies well inside cached territory.
  737. +_LONG_PROMPT = " ".join(
  738. + f"Entry {i}: the quick brown fox number {i} jumps over the lazy dog number {i + 1}."
  739. + for i in range(700)
  740. +)
  741. +_ESSAY_QUESTION = (
  742. + "\n\nWrite a detailed essay on how foxes and dogs are portrayed in folklore, "
  743. + "with examples from at least three cultures."
  744. +)
  745. +
  746. +
  747. +class TestDFlashDraftKVRatio(CustomTestCase, GSM8KMixin):
  748. + model = TARGET_MODEL
  749. + draft_model = DRAFT_MODEL
  750. + page_size = 1
  751. + gsm8k_accuracy_thres = 0.80
  752. + gsm8k_accept_length_thres = 3.0
  753. + gsm8k_num_questions = 200
  754. +
  755. + @classmethod
  756. + def setUpClass(cls):
  757. + cls.base_url = DEFAULT_URL_FOR_TEST
  758. + cls.log_dir = tempfile.mkdtemp(prefix="dflash_draft_kv_ratio_")
  759. + cls.stdout = open(os.path.join(cls.log_dir, "server_stdout.log"), "w")
  760. + cls.stderr = open(os.path.join(cls.log_dir, "server_stderr.log"), "w")
  761. + launch_args = [
  762. + "--trust-remote-code",
  763. + "--speculative-algorithm",
  764. + "DFLASH",
  765. + "--speculative-draft-model-path",
  766. + cls.draft_model,
  767. + "--speculative-num-draft-tokens",
  768. + str(DRAFT_BLOCK_SIZE),
  769. + "--speculative-draft-window-size",
  770. + str(DRAFT_WINDOW),
  771. + "--speculative-draft-kv-ratio",
  772. + str(DRAFT_KV_RATIO),
  773. + "--max-running-requests",
  774. + str(MAX_RUNNING_REQUESTS),
  775. + "--page-size",
  776. + str(cls.page_size),
  777. + "--mem-fraction-static",
  778. + "0.8",
  779. + ]
  780. + with ExitStack() as stack:
  781. + for env, value in (
  782. + (envs.SGLANG_ENABLE_STRICT_MEM_CHECK_DURING_BUSY, 1),
  783. + (envs.SGLANG_ENABLE_ASYNC_ASSERT, True),
  784. + ):
  785. + stack.enter_context(env.override(value))
  786. + cls.process = popen_launch_server(
  787. + cls.model,
  788. + cls.base_url,
  789. + timeout=max(DEFAULT_TIMEOUT_FOR_SERVER_LAUNCH, 1800),
  790. + other_args=launch_args,
  791. + return_stdout_stderr=(cls.stdout, cls.stderr),
  792. + )
  793. +
  794. + @classmethod
  795. + def tearDownClass(cls):
  796. + if hasattr(cls, "process") and cls.process:
  797. + kill_process_tree(cls.process.pid)
  798. + for f in (getattr(cls, "stdout", None), getattr(cls, "stderr", None)):
  799. + if f is not None:
  800. + f.close()
  801. +
  802. + def _server_log(self) -> str:
  803. + self.stdout.flush()
  804. + self.stderr.flush()
  805. + text = ""
  806. + for name in ("server_stdout.log", "server_stderr.log"):
  807. + with open(os.path.join(self.log_dir, name)) as f:
  808. + text += f.read()
  809. + return text
  810. +
  811. + def _generate(self, prompt: str, max_new_tokens: int) -> dict:
  812. + res = requests.post(
  813. + self.base_url + "/generate",
  814. + json={
  815. + "text": prompt,
  816. + "sampling_params": {"max_new_tokens": max_new_tokens, "temperature": 0},
  817. + },
  818. + )
  819. + res.raise_for_status()
  820. + return res.json()
  821. +
  822. + def _flush_cache(self):
  823. + requests.post(self.base_url + "/flush_cache").raise_for_status()
  824. +
  825. + def _pool_sizing(self) -> tuple:
  826. + """(window, ratio, draft tokens, request cap) from the startup log."""
  827. + sizing = re.search(
  828. + r"DFLASH draft KV pool: window=(\d+), ratio=([0-9.]+), tokens=(\d+) "
  829. + r"\(request cap (\d+)",
  830. + self._server_log(),
  831. + )
  832. + self.assertIsNotNone(sizing, "draft KV pool sizing not logged")
  833. + window, ratio, tokens, cap = sizing.groups()
  834. + return int(window), float(ratio), int(tokens), int(cap)
  835. +
  836. + def test_draft_pool_is_ratio_of_target(self):
  837. + window, ratio, draft_tokens, cap = self._pool_sizing()
  838. + self.assertEqual(window, DRAFT_WINDOW)
  839. + self.assertEqual(ratio, DRAFT_KV_RATIO)
  840. + self.assertGreaterEqual(draft_tokens, cap)
  841. + log = self._server_log()
  842. + target_tokens = int(re.search(r"max_total_num_tokens=(\d+)", log).group(1))
  843. + self.assertEqual(
  844. + draft_tokens,
  845. + int(target_tokens * DRAFT_KV_RATIO) // self.page_size * self.page_size,
  846. + )
  847. + # The draft pool is built with that many slots.
  848. + pools = re.findall(r"SWAKVPool .*swa size: (\d+), full size: (\d+)", log)
  849. + self.assertIn(draft_tokens, [int(swa) for swa, _ in pools])
  850. +
  851. + def test_prefix_hit_reuses_whole_prompt(self):
  852. + """A cached prefix keeps its trailing window, so nothing is held back."""
  853. + self._flush_cache()
  854. + cold = self._generate(_LONG_PROMPT, 8)
  855. + prompt_tokens = cold["meta_info"]["prompt_tokens"]
  856. + self.assertGreater(prompt_tokens, 2 * DRAFT_WINDOW)
  857. + warm = self._generate(_LONG_PROMPT, 8)
  858. + cached_tokens = warm["meta_info"]["cached_tokens"]
  859. + print(
  860. + f"prefix hit: prompt_tokens={prompt_tokens}, cached_tokens={cached_tokens}"
  861. + )
  862. + self.assertGreater(cached_tokens, prompt_tokens - DRAFT_WINDOW)
  863. + self.assertGreaterEqual(
  864. + cached_tokens, prompt_tokens - CHECKPOINT_SLACK - self.page_size
  865. + )
  866. +
  867. + def test_greedy_determinism_across_hit(self):
  868. + self._flush_cache()
  869. + prompt = _LONG_PROMPT + _ESSAY_QUESTION
  870. + cold = self._generate(prompt, 256)
  871. + warm = self._generate(prompt, 256)
  872. + self.assertGreater(warm["meta_info"]["cached_tokens"], 0)
  873. + self.assertEqual(cold["text"], warm["text"])
  874. + self.assertIsNone(self.process.poll())
  875. +
  876. + def test_evicted_window_falls_back(self):
  877. + """More cached prefixes than the draft pool holds: the oldest loses its
  878. + window, and a hit on it re-prefills instead of reading freed slots."""
  879. + self._flush_cache()
  880. + _, _, draft_tokens, cap = self._pool_sizing()
  881. + # One more than the pool holds, at the window each cached prefix keeps.
  882. + num_prompts = (draft_tokens - cap) // DRAFT_WINDOW + 2
  883. + prompts = [
  884. + _LONG_PROMPT.replace("quick", f"quick-{i}") + _ESSAY_QUESTION
  885. + for i in range(num_prompts)
  886. + ]
  887. + cold = self._generate(prompts[0], 64)
  888. + prompt_tokens = cold["meta_info"]["prompt_tokens"]
  889. + for prompt in prompts[1:]:
  890. + self._generate(prompt, 8)
  891. + warm = self._generate(prompts[0], 64)
  892. + cached_tokens = warm["meta_info"]["cached_tokens"]
  893. + print(
  894. + f"after {num_prompts} prefixes: prompt_tokens={prompt_tokens}, "
  895. + f"cached_tokens={cached_tokens}"
  896. + )
  897. + self.assertLessEqual(cached_tokens, prompt_tokens - DRAFT_WINDOW)
  898. + self.assertEqual(cold["text"], warm["text"])
  899. + self.assertIsNone(self.process.poll())
  900. +
  901. +
  902. +class TestDFlashDraftKVRatioPaged(TestDFlashDraftKVRatio):
  903. + """The same pool at page size 64, where the allocator pages both sides and
  904. + the draft backend builds its page table from translated slots."""
  905. +
  906. + page_size = 64
  907. +
  908. + def test_gsm8k(self):
  909. + self.skipTest("accuracy does not depend on the page size")
  910. +
  911. +
  912. +if __name__ == "__main__":
  913. + unittest.main()
  914. diff --git a/test/registered/unit/model_executor/test_pool_configurator.py b/test/registered/unit/model_executor/test_pool_configurator.py
  915. index 26fc85b22ae2..3a6c61503385 100644
  916. --- a/test/registered/unit/model_executor/test_pool_configurator.py
  917. +++ b/test/registered/unit/model_executor/test_pool_configurator.py
  918. @@ -96,6 +96,8 @@ def _make_model_runner(
  919. qk_rope_head_dim=64,
  920. swa_kv_lora_rank=128,
  921. swa_qk_rope_head_dim=32,
  922. + speculative_draft_window_size=None,
  923. + speculative_draft_kv_ratio=1.0,
  924. ):
  925. """Create a mock ModelRunner with the fields configurators need."""
  926. mr = MagicMock()
  927. @@ -156,6 +158,8 @@ def _make_model_runner(
  928. speculative_algorithm=speculative_algorithm,
  929. speculative_num_steps=speculative_num_steps,
  930. speculative_eagle_topk=speculative_eagle_topk,
  931. + speculative_draft_window_size=speculative_draft_window_size,
  932. + speculative_draft_kv_ratio=speculative_draft_kv_ratio,
  933. disaggregation_mode=disaggregation_mode,
  934. max_running_requests=max_running_requests,
  935. disaggregation_decode_extra_slots=disaggregation_decode_extra_slots,
  936. @@ -180,11 +184,13 @@ def _make_model_runner(
  937. )
  938. mr.attn_dp_size = 1
  939. mr.pp_size = 1
  940. + mr.draft_kv_ratio = speculative_draft_kv_ratio
  941. mr.pp_group = SimpleNamespace(rank_in_group=0)
  942. mr.spec_aux_config = SimpleNamespace(
  943. eagle_draft_num_layers=None,
  944. eagle_draft_swa_num_layers=None,
  945. dflash_draft_num_layers=None,
  946. + dflash_draft_cell_size_per_token=None,
  947. )
  948. return mr
  949. @@ -1335,5 +1341,115 @@ def test_dsv4_fp8_mtp_swa_ring_keeps_spec_inflation(self):
  950. self.assertEqual(cfg._fixed_swa_bytes(mrr), int(target * cfg._spec_infl))
  951. +class TestDFlashDraftKVRatioConfigurator(CustomTestCase):
  952. + """--speculative-draft-kv-ratio: the draft holds its share of the target's
  953. + tokens, priced into the cell size, and must still cover running requests."""
  954. +
  955. + WINDOW = 64
  956. + DRAFT_CELL = 48 # bytes/token of the draft KV
  957. + MAX_RUNNING = 2
  958. +
  959. + def _make(self, ratio, **kwargs):
  960. + mr = _make_model_runner(
  961. + self,
  962. + num_layers=4,
  963. + speculative_algorithm="DFLASH",
  964. + speculative_num_steps=1,
  965. + speculative_eagle_topk=1,
  966. + speculative_num_draft_tokens=8,
  967. + disable_overlap_schedule=True,
  968. + max_running_requests=self.MAX_RUNNING,
  969. + chunked_prefill_size=32,
  970. + speculative_draft_window_size=self.WINDOW,
  971. + speculative_draft_kv_ratio=ratio,
  972. + **kwargs,
  973. + )
  974. + mr.spec_algorithm.is_dflash_family.return_value = True
  975. + mr.spec_algorithm.is_none.return_value = False
  976. + mr.spec_aux_config.dflash_draft_num_layers = 5
  977. + mr.spec_aux_config.dflash_draft_cell_size_per_token = self.DRAFT_CELL
  978. + return mr
  979. +
  980. + def _run(self, ratio, available_bytes=10_000_000, page_size=1, **kwargs):
  981. + mr = self._make(ratio, page_size=page_size, **kwargs)
  982. + with mock_cpu_env():
  983. + from sglang.srt.model_executor.pool_configurator import (
  984. + create_memory_pool_configurator,
  985. + )
  986. +
  987. + cfg = create_memory_pool_configurator(mr)
  988. + config = cfg.calculate_pool_sizes(available_bytes, page_size)
  989. + return mr, cfg, config
  990. +
  991. + def test_draft_pool_is_ratio_of_target(self):
  992. + _, _, config = self._run(0.5)
  993. + self.assertEqual(config.full_max_total_num_tokens, config.max_total_num_tokens)
  994. + self.assertEqual(
  995. + config.swa_max_total_num_tokens, int(config.max_total_num_tokens * 0.5)
  996. + )
  997. +
  998. + def test_ratio_is_priced_into_cell_size(self):
  999. + mr, cfg, _ = self._run(0.5)
  1000. + target_cell = _full_per_token(mr) * 4
  1001. + self.assertEqual(cfg._cell_size, target_cell + self.DRAFT_CELL // 2)
  1002. +
  1003. + def test_memory_utilization(self):
  1004. + available = 10_000_000
  1005. + mr, _, config = self._run(0.5, available_bytes=available)
  1006. + used = (
  1007. + config.max_total_num_tokens * _full_per_token(mr) * 4
  1008. + + config.swa_max_total_num_tokens * self.DRAFT_CELL
  1009. + )
  1010. + self.assertLessEqual(used, available)
  1011. + self.assertGreater(used, available * 0.99)
  1012. +
  1013. + def test_page_alignment(self):
  1014. + _, _, config = self._run(0.5, page_size=16)
  1015. + self.assertEqual(config.max_total_num_tokens % 16, 0)
  1016. + self.assertEqual(config.swa_max_total_num_tokens % 16, 0)
  1017. +
  1018. + def test_constraint_keeps_ratio(self):
  1019. + _, cfg, _ = self._run(0.5)
  1020. + with mock_cpu_env():
  1021. + constrained = cfg.calculate_pool_sizes_from_max_tokens(1000, page_size=1)
  1022. + self.assertEqual(constrained.max_total_num_tokens, 1000)
  1023. + self.assertEqual(constrained.swa_max_total_num_tokens, 500)
  1024. +
  1025. + def test_ratio_below_request_cap_raises_a_usable_minimum(self):
  1026. + """The minimum the error names must itself pass, which the ratio the
  1027. + target pool was sized at does not give. This ratio also rounds the
  1028. + draft's share below one byte, which the pool still prices at one."""
  1029. + import re
  1030. +
  1031. + with self.assertRaises(RuntimeError) as raised:
  1032. + self._run(0.001)
  1033. + message = str(raised.exception)
  1034. + self.assertRegex(message, r"below the \d+ its running requests need")
  1035. + minimum = float(re.search(r"Use at least ([0-9.]+)", message).group(1))
  1036. + _, _, config = self._run(minimum)
  1037. + self.assertGreaterEqual(
  1038. + config.swa_max_total_num_tokens,
  1039. + int(config.max_total_num_tokens * minimum) - 1,
  1040. + )
  1041. +
  1042. + def test_ratio_of_one_leaves_the_pool_config_alone(self):
  1043. + mr, cfg, config = self._run(1.0)
  1044. + self.assertIsNone(config.swa_max_total_num_tokens)
  1045. + self.assertEqual(cfg._cell_size, _full_per_token(mr) * 4 + self.DRAFT_CELL)
  1046. +
  1047. + def test_unresolved_draft_cell_size_raises(self):
  1048. + """Without the draft's bytes/token the budget scales by layer count,
  1049. + which prices neither the draft's geometry nor the ratio."""
  1050. + mr = self._make(0.25, page_size=1)
  1051. + mr.spec_aux_config.dflash_draft_cell_size_per_token = None
  1052. + with mock_cpu_env():
  1053. + from sglang.srt.model_executor.pool_configurator import (
  1054. + create_memory_pool_configurator,
  1055. + )
  1056. +
  1057. + with self.assertRaisesRegex(ValueError, "KV bytes/token"):
  1058. + create_memory_pool_configurator(mr)
  1059. +
  1060. +
  1061. if __name__ == "__main__":
  1062. unittest.main()
  1063. diff --git a/test/registered/unit/server_args/test_server_args.py b/test/registered/unit/server_args/test_server_args.py
  1064. index fed3493e58c6..d70a80bd35d0 100644
  1065. --- a/test/registered/unit/server_args/test_server_args.py
  1066. +++ b/test/registered/unit/server_args/test_server_args.py
  1067. @@ -1038,6 +1038,24 @@ def test_sm100_native_nvfp4_rejects_prefix_commit_spec_algorithms(self):
  1068. with self.assertRaisesRegex(ValueError, algorithm):
  1069. handle_kv4_compatibility(args)
  1070. + def test_draft_kv_ratio_range(self):
  1071. + """The draft pool is indexed by target slot, so a ratio above 1 would
  1072. + hold slots the target cannot address."""
  1073. + from sglang.srt.arg_groups.speculative_hook import _handle_dflash
  1074. +
  1075. + for ratio in (0, -0.5, 1.5):
  1076. + with self.subTest(ratio=ratio):
  1077. + args = ServerArgs(
  1078. + model_path="dummy",
  1079. + device="cuda",
  1080. + speculative_algorithm="DFLASH",
  1081. + speculative_draft_model_path="dummy-draft",
  1082. + speculative_num_draft_tokens=8,
  1083. + speculative_draft_kv_ratio=ratio,
  1084. + )
  1085. + with self.assertRaisesRegex(ValueError, r"must be in \(0, 1\]"):
  1086. + _handle_dflash(args)
  1087. +
  1088. @override_platform(is_cuda=True, is_sm100=False, is_sm120=True)
  1089. def test_sm120_xqa_keeps_existing_speculative_support(self):
  1090. args = self._make_nvfp4_args(speculative_algorithm="EAGLE")