|
|
@@ -63,7 +63,7 @@ args = [
|
|
|
"--enable-auto-tool-choice",
|
|
|
"--gpu-memory-utilization", "0.87",
|
|
|
"--speculative-config", """
|
|
|
-'{"method":"dflash","model":"{{dflash2}}","num_speculative_tokens":7,"rejection_sample_method":"standard","attention_backend":"B12X","draft_load_config":{"load_format":"auto"}}'""",
|
|
|
+'{"method":"dflash","model":"{{dflash2}}","num_speculative_tokens":7,"rejection_sample_method":"standard","attention_backend":"FLASH_ATTN","draft_load_config":{"load_format":"auto"}}'""",
|
|
|
]
|
|
|
|
|
|
# DFlash2 drafter (A/B vs the card's embedded MTP head): distilled against the
|